diff --git a/.github/ISSUE_TEMPLATE/11-language-change.yml b/.github/ISSUE_TEMPLATE/11-language-change.yml index 37ba2d7e..ac26ef6c 100644 --- a/.github/ISSUE_TEMPLATE/11-language-change.yml +++ b/.github/ISSUE_TEMPLATE/11-language-change.yml @@ -1,7 +1,7 @@ name: Language Change Proposals description: Changes to the language -labels: ["Proposal", "v2", "LanguageChange"] -title: "proposal: Go 2: proposal title" +labels: ["Proposal", "LanguageChange", "LanguageChangeReview"] +title: "proposal: spec: proposal title" body: diff --git a/.gitignore b/.gitignore index 7978d68b..c6512e64 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,7 @@ _testmain.go /src/go/build/zcgo.go /src/go/doc/headscan /src/internal/buildcfg/zbootstrap.go -/src/runtime/internal/sys/zversion.go +/src/internal/runtime/sys/zversion.go /src/unicode/maketables /src/time/tzdata/zzipdata.go /test.out diff --git a/LICENSE b/LICENSE index 6a66aea5..2a7cf70d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/VERSION b/VERSION index 1734c2d8..a4380700 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -go1.23.6 -time 2025-01-31T18:38:03Z +go1.24.0 +time 2025-02-10T23:33:55Z diff --git a/api/go1.24.txt b/api/go1.24.txt new file mode 100644 index 00000000..05e2006e --- /dev/null +++ b/api/go1.24.txt @@ -0,0 +1,223 @@ +pkg bytes, func FieldsFuncSeq([]uint8, func(int32) bool) iter.Seq[[]uint8] #61901 +pkg bytes, func FieldsSeq([]uint8) iter.Seq[[]uint8] #61901 +pkg bytes, func Lines([]uint8) iter.Seq[[]uint8] #61901 +pkg bytes, func SplitAfterSeq([]uint8, []uint8) iter.Seq[[]uint8] #61901 +pkg bytes, func SplitSeq([]uint8, []uint8) iter.Seq[[]uint8] #61901 +pkg crypto/cipher, func NewCFBDecrypter //deprecated #69445 +pkg crypto/cipher, func NewCFBEncrypter //deprecated #69445 +pkg crypto/cipher, func NewGCMWithRandomNonce(Block) (AEAD, error) #69981 +pkg crypto/cipher, func NewOFB //deprecated #69445 +pkg crypto/fips140, func Enabled() bool #70123 +pkg crypto/hkdf, func Expand[$0 hash.Hash](func() $0, []uint8, string, int) ([]uint8, error) #61477 +pkg crypto/hkdf, func Extract[$0 hash.Hash](func() $0, []uint8, []uint8) ([]uint8, error) #61477 +pkg crypto/hkdf, func Key[$0 hash.Hash](func() $0, []uint8, []uint8, string, int) ([]uint8, error) #61477 +pkg crypto/mlkem, const CiphertextSize1024 = 1568 #70122 +pkg crypto/mlkem, const CiphertextSize1024 ideal-int #70122 +pkg crypto/mlkem, const CiphertextSize768 = 1088 #70122 +pkg crypto/mlkem, const CiphertextSize768 ideal-int #70122 +pkg crypto/mlkem, const EncapsulationKeySize1024 = 1568 #70122 +pkg crypto/mlkem, const EncapsulationKeySize1024 ideal-int #70122 +pkg crypto/mlkem, const EncapsulationKeySize768 = 1184 #70122 +pkg crypto/mlkem, const EncapsulationKeySize768 ideal-int #70122 +pkg crypto/mlkem, const SeedSize = 64 #70122 +pkg crypto/mlkem, const SeedSize ideal-int #70122 +pkg crypto/mlkem, const SharedKeySize = 32 #70122 +pkg crypto/mlkem, const SharedKeySize ideal-int #70122 +pkg crypto/mlkem, func GenerateKey1024() (*DecapsulationKey1024, error) #70122 +pkg crypto/mlkem, func GenerateKey768() (*DecapsulationKey768, error) #70122 +pkg crypto/mlkem, func NewDecapsulationKey1024([]uint8) (*DecapsulationKey1024, error) #70122 +pkg crypto/mlkem, func NewDecapsulationKey768([]uint8) (*DecapsulationKey768, error) #70122 +pkg crypto/mlkem, func NewEncapsulationKey1024([]uint8) (*EncapsulationKey1024, error) #70122 +pkg crypto/mlkem, func NewEncapsulationKey768([]uint8) (*EncapsulationKey768, error) #70122 +pkg crypto/mlkem, method (*DecapsulationKey1024) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*DecapsulationKey1024) Decapsulate([]uint8) ([]uint8, error) #70122 +pkg crypto/mlkem, method (*DecapsulationKey1024) EncapsulationKey() *EncapsulationKey1024 #70122 +pkg crypto/mlkem, method (*DecapsulationKey768) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*DecapsulationKey768) Decapsulate([]uint8) ([]uint8, error) #70122 +pkg crypto/mlkem, method (*DecapsulationKey768) EncapsulationKey() *EncapsulationKey768 #70122 +pkg crypto/mlkem, method (*EncapsulationKey1024) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*EncapsulationKey1024) Encapsulate() ([]uint8, []uint8) #70122 +pkg crypto/mlkem, method (*EncapsulationKey768) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*EncapsulationKey768) Encapsulate() ([]uint8, []uint8) #70122 +pkg crypto/mlkem, type DecapsulationKey1024 struct #70122 +pkg crypto/mlkem, type DecapsulationKey768 struct #70122 +pkg crypto/mlkem, type EncapsulationKey1024 struct #70122 +pkg crypto/mlkem, type EncapsulationKey768 struct #70122 +pkg crypto/pbkdf2, func Key[$0 hash.Hash](func() $0, string, []uint8, int, int) ([]uint8, error) #69488 +pkg crypto/rand, func Text() string #67057 +pkg crypto/sha3, func New224() *SHA3 #69982 +pkg crypto/sha3, func New256() *SHA3 #69982 +pkg crypto/sha3, func New384() *SHA3 #69982 +pkg crypto/sha3, func New512() *SHA3 #69982 +pkg crypto/sha3, func NewCSHAKE128([]uint8, []uint8) *SHAKE #69982 +pkg crypto/sha3, func NewCSHAKE256([]uint8, []uint8) *SHAKE #69982 +pkg crypto/sha3, func NewSHAKE128() *SHAKE #69982 +pkg crypto/sha3, func NewSHAKE256() *SHAKE #69982 +pkg crypto/sha3, func Sum224([]uint8) [28]uint8 #69982 +pkg crypto/sha3, func Sum256([]uint8) [32]uint8 #69982 +pkg crypto/sha3, func Sum384([]uint8) [48]uint8 #69982 +pkg crypto/sha3, func Sum512([]uint8) [64]uint8 #69982 +pkg crypto/sha3, func SumSHAKE128([]uint8, int) []uint8 #69982 +pkg crypto/sha3, func SumSHAKE256([]uint8, int) []uint8 #69982 +pkg crypto/sha3, method (*SHA3) AppendBinary([]uint8) ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHA3) BlockSize() int #69982 +pkg crypto/sha3, method (*SHA3) MarshalBinary() ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHA3) Reset() #69982 +pkg crypto/sha3, method (*SHA3) Size() int #69982 +pkg crypto/sha3, method (*SHA3) Sum([]uint8) []uint8 #69982 +pkg crypto/sha3, method (*SHA3) UnmarshalBinary([]uint8) error #69982 +pkg crypto/sha3, method (*SHA3) Write([]uint8) (int, error) #69982 +pkg crypto/sha3, method (*SHAKE) AppendBinary([]uint8) ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHAKE) BlockSize() int #69982 +pkg crypto/sha3, method (*SHAKE) MarshalBinary() ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHAKE) Read([]uint8) (int, error) #69982 +pkg crypto/sha3, method (*SHAKE) Reset() #69982 +pkg crypto/sha3, method (*SHAKE) UnmarshalBinary([]uint8) error #69982 +pkg crypto/sha3, method (*SHAKE) Write([]uint8) (int, error) #69982 +pkg crypto/sha3, type SHA3 struct #69982 +pkg crypto/sha3, type SHAKE struct #69982 +pkg crypto/subtle, func WithDataIndependentTiming(func()) #66450 +pkg crypto/tls, const X25519MLKEM768 = 4588 #69985 +pkg crypto/tls, const X25519MLKEM768 CurveID #69985 +pkg crypto/tls, type ClientHelloInfo struct, Extensions []uint16 #32936 +pkg crypto/tls, type Config struct, EncryptedClientHelloKeys []EncryptedClientHelloKey #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct, Config []uint8 #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct, PrivateKey []uint8 #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct, SendAsRetry bool #68500 +pkg crypto/x509, const NoValidChains = 10 #68484 +pkg crypto/x509, const NoValidChains InvalidReason #68484 +pkg crypto/x509, method (OID) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg crypto/x509, method (OID) AppendText([]uint8) ([]uint8, error) #62384 +pkg crypto/x509, type Certificate struct, InhibitAnyPolicy int #68484 +pkg crypto/x509, type Certificate struct, InhibitAnyPolicyZero bool #68484 +pkg crypto/x509, type Certificate struct, InhibitPolicyMapping int #68484 +pkg crypto/x509, type Certificate struct, InhibitPolicyMappingZero bool #68484 +pkg crypto/x509, type Certificate struct, PolicyMappings []PolicyMapping #68484 +pkg crypto/x509, type Certificate struct, RequireExplicitPolicy int #68484 +pkg crypto/x509, type Certificate struct, RequireExplicitPolicyZero bool #68484 +pkg crypto/x509, type PolicyMapping struct #68484 +pkg crypto/x509, type PolicyMapping struct, IssuerDomainPolicy OID #68484 +pkg crypto/x509, type PolicyMapping struct, SubjectDomainPolicy OID #68484 +pkg crypto/x509, type VerifyOptions struct, CertificatePolicies []OID #68484 +pkg debug/elf, const VER_FLG_BASE = 1 #63952 +pkg debug/elf, const VER_FLG_BASE DynamicVersionFlag #63952 +pkg debug/elf, const VER_FLG_INFO = 4 #63952 +pkg debug/elf, const VER_FLG_INFO DynamicVersionFlag #63952 +pkg debug/elf, const VER_FLG_WEAK = 2 #63952 +pkg debug/elf, const VER_FLG_WEAK DynamicVersionFlag #63952 +pkg debug/elf, method (*File) DynamicVersionNeeds() ([]DynamicVersionNeed, error) #63952 +pkg debug/elf, method (*File) DynamicVersions() ([]DynamicVersion, error) #63952 +pkg debug/elf, type DynamicVersion struct #63952 +pkg debug/elf, type DynamicVersion struct, Deps []string #63952 +pkg debug/elf, type DynamicVersion struct, Flags DynamicVersionFlag #63952 +pkg debug/elf, type DynamicVersion struct, Name string #63952 +pkg debug/elf, type DynamicVersion struct, Index uint16 #63952 +pkg debug/elf, type DynamicVersionDep struct #63952 +pkg debug/elf, type DynamicVersionDep struct, Dep string #63952 +pkg debug/elf, type DynamicVersionDep struct, Flags DynamicVersionFlag #63952 +pkg debug/elf, type DynamicVersionDep struct, Index uint16 #63952 +pkg debug/elf, type DynamicVersionFlag uint16 #63952 +pkg debug/elf, type DynamicVersionNeed struct #63952 +pkg debug/elf, type DynamicVersionNeed struct, Name string #63952 +pkg debug/elf, type DynamicVersionNeed struct, Needs []DynamicVersionDep #63952 +pkg debug/elf, type Symbol struct, HasVersion bool #63952 +pkg debug/elf, type Symbol struct, VersionIndex VersionIndex #63952 +pkg debug/elf, method (VersionIndex) Index() uint16 #63952 +pkg debug/elf, method (VersionIndex) IsHidden() bool #63952 +pkg debug/elf, type VersionIndex uint16 #63952 +pkg encoding, type BinaryAppender interface { AppendBinary } #62384 +pkg encoding, type BinaryAppender interface, AppendBinary([]uint8) ([]uint8, error) #62384 +pkg encoding, type TextAppender interface { AppendText } #62384 +pkg encoding, type TextAppender interface, AppendText([]uint8) ([]uint8, error) #62384 +pkg go/types, method (*Interface) EmbeddedTypes() iter.Seq[Type] #66626 +pkg go/types, method (*Interface) ExplicitMethods() iter.Seq[*Func] #66626 +pkg go/types, method (*Interface) Methods() iter.Seq[*Func] #66626 +pkg go/types, method (*MethodSet) Methods() iter.Seq[*Selection] #66626 +pkg go/types, method (*Named) Methods() iter.Seq[*Func] #66626 +pkg go/types, method (*Scope) Children() iter.Seq[*Scope] #66626 +pkg go/types, method (*Struct) Fields() iter.Seq[*Var] #66626 +pkg go/types, method (*Tuple) Variables() iter.Seq[*Var] #66626 +pkg go/types, method (*TypeList) Types() iter.Seq[Type] #66626 +pkg go/types, method (*TypeParamList) TypeParams() iter.Seq[*TypeParam] #66626 +pkg go/types, method (*Union) Terms() iter.Seq[*Term] #66626 +pkg hash/maphash, func Comparable[$0 comparable](Seed, $0) uint64 #54670 +pkg hash/maphash, func WriteComparable[$0 comparable](*Hash, $0) #54670 +pkg log/slog, method (*LevelVar) AppendText([]uint8) ([]uint8, error) #62384 +pkg log/slog, method (Level) AppendText([]uint8) ([]uint8, error) #62384 +pkg log/slog, var DiscardHandler Handler #62005 +pkg math/big, method (*Float) AppendText([]uint8) ([]uint8, error) #62384 +pkg math/big, method (*Int) AppendText([]uint8) ([]uint8, error) #62384 +pkg math/big, method (*Rat) AppendText([]uint8) ([]uint8, error) #62384 +pkg math/rand/v2, method (*ChaCha8) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg math/rand/v2, method (*PCG) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net, method (IP) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/http, method (*Protocols) SetHTTP1(bool) #67814 +pkg net/http, method (*Protocols) SetHTTP2(bool) #67814 +pkg net/http, method (*Protocols) SetUnencryptedHTTP2(bool) #67816 +pkg net/http, method (Protocols) HTTP1() bool #67814 +pkg net/http, method (Protocols) HTTP2() bool #67814 +pkg net/http, method (Protocols) String() string #67814 +pkg net/http, method (Protocols) UnencryptedHTTP2() bool #67816 +pkg net/http, type HTTP2Config struct #67813 +pkg net/http, type HTTP2Config struct, CountError func(string) #67813 +pkg net/http, type HTTP2Config struct, MaxConcurrentStreams int #67813 +pkg net/http, type HTTP2Config struct, MaxDecoderHeaderTableSize int #67813 +pkg net/http, type HTTP2Config struct, MaxEncoderHeaderTableSize int #67813 +pkg net/http, type HTTP2Config struct, MaxReadFrameSize int #67813 +pkg net/http, type HTTP2Config struct, MaxReceiveBufferPerConnection int #67813 +pkg net/http, type HTTP2Config struct, MaxReceiveBufferPerStream int #67813 +pkg net/http, type HTTP2Config struct, PermitProhibitedCipherSuites bool #67813 +pkg net/http, type HTTP2Config struct, PingTimeout time.Duration #67813 +pkg net/http, type HTTP2Config struct, SendPingTimeout time.Duration #67813 +pkg net/http, type HTTP2Config struct, WriteByteTimeout time.Duration #67813 +pkg net/http, type Protocols struct #67814 +pkg net/http, type Server struct, HTTP2 *HTTP2Config #67813 +pkg net/http, type Server struct, Protocols *Protocols #67814 +pkg net/http, type Transport struct, HTTP2 *HTTP2Config #67813 +pkg net/http, type Transport struct, Protocols *Protocols #67814 +pkg net/netip, method (Addr) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (Addr) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (AddrPort) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (AddrPort) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (Prefix) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (Prefix) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/url, method (*URL) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg os, func OpenInRoot(string, string) (*File, error) #67002 +pkg os, func OpenRoot(string) (*Root, error) #67002 +pkg os, method (*Root) Close() error #67002 +pkg os, method (*Root) Create(string) (*File, error) #67002 +pkg os, method (*Root) FS() fs.FS #67002 +pkg os, method (*Root) Lstat(string) (fs.FileInfo, error) #67002 +pkg os, method (*Root) Mkdir(string, fs.FileMode) error #67002 +pkg os, method (*Root) Name() string #67002 +pkg os, method (*Root) Open(string) (*File, error) #67002 +pkg os, method (*Root) OpenFile(string, int, fs.FileMode) (*File, error) #67002 +pkg os, method (*Root) OpenRoot(string) (*Root, error) #67002 +pkg os, method (*Root) Remove(string) error #67002 +pkg os, method (*Root) Stat(string) (fs.FileInfo, error) #67002 +pkg os, type Root struct #67002 +pkg regexp, method (*Regexp) AppendText([]uint8) ([]uint8, error) #62384 +pkg runtime, func AddCleanup[$0 interface{}, $1 interface{}](*$0, func($1), $1) Cleanup #67535 +pkg runtime, func GOROOT //deprecated #51473 +pkg runtime, method (Cleanup) Stop() #67535 +pkg runtime, type Cleanup struct #67535 +pkg strings, func FieldsFuncSeq(string, func(int32) bool) iter.Seq[string] #61901 +pkg strings, func FieldsSeq(string) iter.Seq[string] #61901 +pkg strings, func Lines(string) iter.Seq[string] #61901 +pkg strings, func SplitAfterSeq(string, string) iter.Seq[string] #61901 +pkg strings, func SplitSeq(string, string) iter.Seq[string] #61901 +pkg testing, method (*B) Chdir(string) #62516 +pkg testing, method (*B) Context() context.Context #36532 +pkg testing, method (*B) Loop() bool #61515 +pkg testing, method (*F) Chdir(string) #62516 +pkg testing, method (*F) Context() context.Context #36532 +pkg testing, method (*T) Chdir(string) #62516 +pkg testing, method (*T) Context() context.Context #36532 +pkg testing, type TB interface, Chdir(string) #62516 +pkg testing, type TB interface, Context() context.Context #36532 +pkg time, method (Time) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg time, method (Time) AppendText([]uint8) ([]uint8, error) #62384 +pkg weak, func Make[$0 interface{}](*$0) Pointer[$0] #67552 +pkg weak, method (Pointer[$0]) Value() *$0 #67552 +pkg weak, type Pointer[$0 interface{}] struct #67552 diff --git a/codereview.cfg b/codereview.cfg index 3cf4bb2d..e97659c8 100644 --- a/codereview.cfg +++ b/codereview.cfg @@ -1,2 +1,2 @@ -branch: release-branch.go1.23 +branch: release-branch.go1.24 parent-branch: master diff --git a/doc/README.md b/doc/README.md index b0d99456..b7aefc8d 100644 --- a/doc/README.md +++ b/doc/README.md @@ -70,6 +70,6 @@ To begin the next release development cycle, populate the contents of `next` with those of `initial`. From the repo root: > cd doc - > cp -r initial/* next + > cp -R initial/ next Then edit `next/1-intro.md` to refer to the next version. diff --git a/doc/go1.17_spec.html b/doc/go1.17_spec.html index 9f408bcc..dbff3598 100644 --- a/doc/go1.17_spec.html +++ b/doc/go1.17_spec.html @@ -1,6 +1,6 @@ diff --git a/doc/go_mem.html b/doc/go_mem.html index c0b81d3f..633d35cd 100644 --- a/doc/go_mem.html +++ b/doc/go_mem.html @@ -82,7 +82,7 @@ while still insisting that races are errors and that tools can diagnose and repo

The following formal definition of Go's memory model closely follows the approach presented by Hans-J. Boehm and Sarita V. Adve in -“Foundations of the C++ Concurrency Memory Model”, +“Foundations of the C++ Concurrency Memory Model”, published in PLDI 2008. The definition of data-race-free programs and the guarantee of sequential consistency for race-free programs are equivalent to the ones in that work. diff --git a/doc/go_spec.html b/doc/go_spec.html index b5b7f15b..db5fba45 100644 --- a/doc/go_spec.html +++ b/doc/go_spec.html @@ -1,6 +1,6 @@ @@ -798,7 +798,6 @@ If a variable has not yet been assigned a value, its value is the zero value for its type.

-

Types

@@ -810,12 +809,12 @@ from existing types.

-Type      = TypeName [ TypeArgs ] | TypeLit | "(" Type ")" .
-TypeName  = identifier | QualifiedIdent .
-TypeArgs  = "[" TypeList [ "," ] "]" .
-TypeList  = Type { "," Type } .
-TypeLit   = ArrayType | StructType | PointerType | FunctionType | InterfaceType |
-            SliceType | MapType | ChannelType .
+Type     = TypeName [ TypeArgs ] | TypeLit | "(" Type ")" .
+TypeName = identifier | QualifiedIdent .
+TypeArgs = "[" TypeList [ "," ] "]" .
+TypeList = Type { "," Type } .
+TypeLit  = ArrayType | StructType | PointerType | FunctionType | InterfaceType |
+           SliceType | MapType | ChannelType .
 

@@ -1086,7 +1085,7 @@ A field declared with a type but no explicit field name is called an embedded An embedded field must be specified as a type name T or as a pointer to a non-interface type name *T, and T itself may not be -a pointer type. The unqualified type name acts as the field name. +a pointer type or type parameter. The unqualified type name acts as the field name.

@@ -1127,7 +1126,7 @@ of a struct except that they cannot be used as field names in
 

-Given a struct type S and a named type +Given a struct type S and a type name T, promoted methods are included in the method set of the struct as follows:

+ +

Go 1.24

+ +

Type unification rules

diff --git a/doc/godebug.md b/doc/godebug.md index b3a43664..1b5674f2 100644 --- a/doc/godebug.md +++ b/doc/godebug.md @@ -34,6 +34,7 @@ For example, if a Go program is running in an environment that contains then that Go program will disable the use of HTTP/2 by default in both the HTTP client and the HTTP server. +Unrecognized settings in the `GODEBUG` environment variable are ignored. It is also possible to set the default `GODEBUG` for a given program (discussed below). @@ -150,6 +151,68 @@ for example, see the [runtime documentation](/pkg/runtime#hdr-Environment_Variables) and the [go command documentation](/cmd/go#hdr-Build_and_test_caching). +### Go 1.24 + +Go 1.24 changed the global [`math/rand.Seed`](/pkg/math/rand/#Seed) to be a +no-op. This behavior is controlled by the `randseednop` setting. +For Go 1.24 it defaults to `randseednop=1`. +Using `randseednop=0` reverts to the pre-Go 1.24 behavior. + +Go 1.24 added new values for the `multipathtcp` setting. +The possible values for `multipathtcp` are now: +- "0": disable MPTCP on dialers and listeners by default +- "1": enable MPTCP on dialers and listeners by default +- "2": enable MPTCP on listeners only by default +- "3": enable MPTCP on dialers only by default + +For Go 1.24, it now defaults to multipathtcp="2", thus +enabled by default on listeners. Using multipathtcp="0" reverts to the +pre-Go 1.24 behavior. + +Go 1.24 changed the behavior of `go test -json` to emit build errors as JSON +instead of text. +These new JSON events are distinguished by new `Action` values, +but can still cause problems with CI systems that aren't robust to these events. +This behavior can be controlled with the `gotestjsonbuildtext` setting. +Using `gotestjsonbuildtext=1` restores the 1.23 behavior. +This setting will be removed in a future release, Go 1.28 at the earliest. + +Go 1.24 changed [`crypto/rsa`](/pkg/crypto/rsa) to require RSA keys to be at +least 1024 bits. This behavior can be controlled with the `rsa1024min` setting. +Using `rsa1024min=0` restores the Go 1.23 behavior. + +Go 1.24 introduced a mechanism for enabling platform specific Data Independent +Timing (DIT) modes in the [`crypto/subtle`](/pkg/crypto/subtle) package. This +mode can be enabled for an entire program with the `dataindependenttiming` setting. +For Go 1.24 it defaults to `dataindependenttiming=0`. There is no change in default +behavior from Go 1.23 when `dataindependenttiming` is unset. +Using `dataindependenttiming=1` enables the DIT mode for the entire Go program. +When enabled, DIT will be enabled when calling into C from Go. When enabled, +calling into Go code from C will enable DIT, and disable it before returning to +C if it was not enabled when Go code was entered. +This currently only affects arm64 programs. For all other platforms it is a no-op. + +Go 1.24 removed the `x509sha1` setting. `crypto/x509` no longer supports verifying +signatures on certificates that use SHA-1 based signature algorithms. + +Go 1.24 changes the default value of the [`x509usepolicies` +setting.](/pkg/crypto/x509/#CreateCertificate) from `0` to `1`. When marshalling +certificates, policies are now taken from the +[`Certificate.Policies`](/pkg/crypto/x509/#Certificate.Policies) field rather +than the +[`Certificate.PolicyIdentifiers`](/pkg/crypto/x509/#Certificate.PolicyIdentifiers) +field by default. + +Go 1.24 enabled the post-quantum key exchange mechanism +X25519MLKEM768 by default. The default can be reverted using the +[`tlsmlkem` setting](/pkg/crypto/tls/#Config.CurvePreferences). +Go 1.24 also removed X25519Kyber768Draft00 and the Go 1.23 `tlskyber` setting. + +Go 1.24 made [`ParsePKCS1PrivateKey`](/pkg/crypto/x509/#ParsePKCS1PrivateKey) +use and validate the CRT parameters in the encoded private key. This behavior +can be controlled with the `x509rsacrt` setting. Using `x509rsacrt=0` restores +the Go 1.23 behavior. + ### Go 1.23 Go 1.23 changed the channels created by package time to be unbuffered @@ -342,7 +405,7 @@ There is no plan to remove this setting. Go 1.18 removed support for SHA1 in most X.509 certificates, controlled by the [`x509sha1` setting](/pkg/crypto/x509#InsecureAlgorithmError). -This setting will be removed in a future release, Go 1.22 at the earliest. +This setting was removed in Go 1.24. ### Go 1.10 diff --git a/doc/initial/1-intro.md b/doc/initial/1-intro.md index 8c9948dd..84ffee85 100644 --- a/doc/initial/1-intro.md +++ b/doc/initial/1-intro.md @@ -1,9 +1,3 @@ - - diff --git a/lib/fips140/Makefile b/lib/fips140/Makefile new file mode 100644 index 00000000..8dcb8fbe --- /dev/null +++ b/lib/fips140/Makefile @@ -0,0 +1,46 @@ +# Copyright 2024 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Rules for building and testing new FIPS snapshots. +# For example: +# +# make v1.2.3.zip +# make v1.2.3.test +# +# and then if changes are needed, check them into master +# and run 'make v1.2.3.rm' and repeat. +# +# Note that once published a snapshot zip file should never +# be modified. We record the sha256 hashes of the zip files +# in fips140.sum, and the cmd/go/internal/fips140 test checks +# that the zips match. +# +# When the zip file is finalized, run 'make updatesum' to update +# fips140.sum. + +default: + @echo nothing to make + +# make v1.2.3.zip builds a v1.2.3.zip file +# from the current origin/master. +# copy and edit the 'go run' command by hand to use a different branch. +v%.zip: + git fetch origin master + go run ../../src/cmd/go/internal/fips140/mkzip.go v$* + +# normally mkzip refuses to overwrite an existing zip file. +# make v1.2.3.rm removes the zip file and and unpacked +# copy from the module cache. +v%.rm: + rm -f v$*.zip + chmod -R u+w $$(go env GOMODCACHE)/golang.org/fips140@v$* 2>/dev/null || true + rm -rf $$(go env GOMODCACHE)/golang.org/fips140@v$* + +# make v1.2.3.test runs the crypto tests using that snapshot. +v%.test: + GOFIPS140=v$* go test -short crypto... + +# make updatesum updates the fips140.sum file. +updatesum: + go test cmd/go/internal/fips140 -update diff --git a/lib/fips140/README.md b/lib/fips140/README.md new file mode 100644 index 00000000..38ca130d --- /dev/null +++ b/lib/fips140/README.md @@ -0,0 +1,9 @@ +This directory holds snapshots of the crypto/internal/fips140 tree +that are being validated and certified for FIPS-140 use. +The file x.txt (for example, inprocess.txt, certified.txt) +defines the meaning of the FIPS version alias x, listing +the exact version to use. + +The zip files are created by cmd/go/internal/fips140/mkzip.go. +The fips140.sum file lists checksums for the zip files. +See the Makefile for recipes. diff --git a/lib/fips140/fips140.sum b/lib/fips140/fips140.sum new file mode 100644 index 00000000..66b1e23d --- /dev/null +++ b/lib/fips140/fips140.sum @@ -0,0 +1,12 @@ +# SHA256 checksums of snapshot zip files in this directory. +# These checksums are included in the FIPS security policy +# (validation instructions sent to the lab) and MUST NOT CHANGE. +# That is, the zip files themselves must not change. +# +# It is okay to add new zip files to the list, and it is okay to +# remove zip files from the list when they are removed from +# this directory. To update this file: +# +# go test cmd/go/internal/fips140 -update +# +v1.0.0.zip b50508feaeff05d22516b21e1fd210bbf5d6a1e422eaf2cfa23fe379342713b8 diff --git a/lib/fips140/v1.0.0.zip b/lib/fips140/v1.0.0.zip new file mode 100644 index 00000000..bd9d3c19 Binary files /dev/null and b/lib/fips140/v1.0.0.zip differ diff --git a/lib/time/update.bash b/lib/time/update.bash index bed82b4f..67cb016e 100755 --- a/lib/time/update.bash +++ b/lib/time/update.bash @@ -24,8 +24,8 @@ # in the CL match the update.bash in the CL. # Versions to use. -CODE=2024a -DATA=2024a +CODE=2025a +DATA=2025a set -e diff --git a/lib/time/zoneinfo.zip b/lib/time/zoneinfo.zip index bb38801b..6ba9ff6f 100644 Binary files a/lib/time/zoneinfo.zip and b/lib/time/zoneinfo.zip differ diff --git a/misc/wasm/go_js_wasm_exec b/lib/wasm/go_js_wasm_exec similarity index 100% rename from misc/wasm/go_js_wasm_exec rename to lib/wasm/go_js_wasm_exec diff --git a/misc/wasm/go_wasip1_wasm_exec b/lib/wasm/go_wasip1_wasm_exec similarity index 100% rename from misc/wasm/go_wasip1_wasm_exec rename to lib/wasm/go_wasip1_wasm_exec diff --git a/misc/wasm/wasm_exec.js b/lib/wasm/wasm_exec.js similarity index 97% rename from misc/wasm/wasm_exec.js rename to lib/wasm/wasm_exec.js index bc6f2102..d71af9e9 100644 --- a/misc/wasm/wasm_exec.js +++ b/lib/wasm/wasm_exec.js @@ -14,7 +14,7 @@ if (!globalThis.fs) { let outputBuf = ""; globalThis.fs = { - constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused + constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1, O_DIRECTORY: -1 }, // unused writeSync(fd, buf) { outputBuf += decoder.decode(buf); const nl = outputBuf.lastIndexOf("\n"); @@ -73,6 +73,14 @@ } } + if (!globalThis.path) { + globalThis.path = { + resolve(...pathSegments) { + return pathSegments.join("/"); + } + } + } + if (!globalThis.crypto) { throw new Error("globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)"); } @@ -208,10 +216,16 @@ return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len)); } + const testCallExport = (a, b) => { + this._inst.exports.testExport0(); + return this._inst.exports.testExport(a, b); + } + const timeOrigin = Date.now() - performance.now(); this.importObject = { _gotest: { add: (a, b) => a + b, + callExport: testCallExport, }, gojs: { // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) diff --git a/misc/wasm/wasm_exec_node.js b/lib/wasm/wasm_exec_node.js similarity index 97% rename from misc/wasm/wasm_exec_node.js rename to lib/wasm/wasm_exec_node.js index 98606908..dd65b198 100644 --- a/misc/wasm/wasm_exec_node.js +++ b/lib/wasm/wasm_exec_node.js @@ -11,6 +11,7 @@ if (process.argv.length < 3) { globalThis.require = require; globalThis.fs = require("fs"); +globalThis.path = require("path"); globalThis.TextEncoder = require("util").TextEncoder; globalThis.TextDecoder = require("util").TextDecoder; diff --git a/misc/wasm/wasm_exec.html b/misc/wasm/wasm_exec.html index 72e64473..694b526d 100644 --- a/misc/wasm/wasm_exec.html +++ b/misc/wasm/wasm_exec.html @@ -17,7 +17,7 @@ license that can be found in the LICENSE file. (see https://caniuse.com/#feat=textencoder) --> - + = min(r0.Max.X, s0.Max.X) || likewiseForY { etc } if r.Empty() { - return ZR + return Rectangle{} } return r } diff --git a/src/image/gif/writer.go b/src/image/gif/writer.go index 0d2a1321..129d0ab2 100644 --- a/src/image/gif/writer.go +++ b/src/image/gif/writer.go @@ -146,8 +146,8 @@ func (e *encoder) writeHeader() { } // Logical screen width and height. - byteorder.LePutUint16(e.buf[0:2], uint16(e.g.Config.Width)) - byteorder.LePutUint16(e.buf[2:4], uint16(e.g.Config.Height)) + byteorder.LEPutUint16(e.buf[0:2], uint16(e.g.Config.Width)) + byteorder.LEPutUint16(e.buf[2:4], uint16(e.g.Config.Height)) e.write(e.buf[:4]) if p, ok := e.g.Config.ColorModel.(color.Palette); ok && len(p) > 0 { @@ -185,7 +185,7 @@ func (e *encoder) writeHeader() { } e.buf[0] = 0x03 // Block Size. e.buf[1] = 0x01 // Sub-block Index. - byteorder.LePutUint16(e.buf[2:4], uint16(e.g.LoopCount)) + byteorder.LEPutUint16(e.buf[2:4], uint16(e.g.LoopCount)) e.buf[4] = 0x00 // Block Terminator. e.write(e.buf[:5]) } @@ -271,7 +271,7 @@ func (e *encoder) writeImageBlock(pm *image.Paletted, delay int, disposal byte) } else { e.buf[3] = 0x00 | disposal<<2 } - byteorder.LePutUint16(e.buf[4:6], uint16(delay)) // Delay Time (1/100ths of a second) + byteorder.LEPutUint16(e.buf[4:6], uint16(delay)) // Delay Time (1/100ths of a second) // Transparent color index. if transparentIndex != -1 { @@ -283,10 +283,10 @@ func (e *encoder) writeImageBlock(pm *image.Paletted, delay int, disposal byte) e.write(e.buf[:8]) } e.buf[0] = sImageDescriptor - byteorder.LePutUint16(e.buf[1:3], uint16(b.Min.X)) - byteorder.LePutUint16(e.buf[3:5], uint16(b.Min.Y)) - byteorder.LePutUint16(e.buf[5:7], uint16(b.Dx())) - byteorder.LePutUint16(e.buf[7:9], uint16(b.Dy())) + byteorder.LEPutUint16(e.buf[1:3], uint16(b.Min.X)) + byteorder.LEPutUint16(e.buf[3:5], uint16(b.Min.Y)) + byteorder.LEPutUint16(e.buf[5:7], uint16(b.Dx())) + byteorder.LEPutUint16(e.buf[7:9], uint16(b.Dy())) e.write(e.buf[:9]) // To determine whether or not this frame's palette is the same as the diff --git a/src/image/jpeg/dct_test.go b/src/image/jpeg/dct_test.go index ed5b73d5..11819db8 100644 --- a/src/image/jpeg/dct_test.go +++ b/src/image/jpeg/dct_test.go @@ -112,12 +112,42 @@ func alpha(i int) float64 { return math.Sqrt2 } -var cosines [32]float64 // cosines[k] = cos(π/2 * k/8) +var cosines = [32]float64{ + +1.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 0) + +0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 1) + +0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 2) + +0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 3) + +0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 4) + +0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 5) + +0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 6) + +0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 7) -func init() { - for k := range cosines { - cosines[k] = math.Cos(math.Pi * float64(k) / 16) - } + -0.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 8) + -0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 9) + -0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 10) + -0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 11) + -0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 12) + -0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 13) + -0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 14) + -0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 15) + + -1.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 16) + -0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 17) + -0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 18) + -0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 19) + -0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 20) + -0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 21) + -0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 22) + -0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 23) + + +0.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 24) + +0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 25) + +0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 26) + +0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 27) + +0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 28) + +0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 29) + +0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 30) + +0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 31) } // slowFDCT performs the 8*8 2-dimensional forward discrete cosine transform: diff --git a/src/image/jpeg/writer.go b/src/image/jpeg/writer.go index 87c109ab..0bda55c6 100644 --- a/src/image/jpeg/writer.go +++ b/src/image/jpeg/writer.go @@ -50,8 +50,8 @@ const ( // unscaledQuant are the unscaled quantization tables in zig-zag order. Each // encoder copies and scales the tables according to its quality parameter. -// The values are derived from section K.1 after converting from natural to -// zig-zag order. +// The values are derived from section K.1 of the spec, after converting from +// natural to zig-zag order. var unscaledQuant = [nQuantIndex][blockSize]byte{ // Luminance. { @@ -89,14 +89,22 @@ const ( // huffmanSpec specifies a Huffman encoding. type huffmanSpec struct { - // count[i] is the number of codes of length i bits. + // count[i] is the number of codes of length i+1 bits. count [16]byte // value[i] is the decoded value of the i'th codeword. value []byte } // theHuffmanSpec is the Huffman encoding specifications. -// This encoder uses the same Huffman encoding for all images. +// +// This encoder uses the same Huffman encoding for all images. It is also the +// same Huffman encoding used by section K.3 of the spec. +// +// The DC tables have 12 decoded values, called categories. +// +// The AC tables have 162 decoded values: bytes that pack a 4-bit Run and a +// 4-bit Size. There are 16 valid Runs and 10 valid Sizes, plus two special R|S +// cases: 0|0 (meaning EOB) and F|0 (meaning ZRL). var theHuffmanSpec = [nHuffIndex]huffmanSpec{ // Luminance DC. { diff --git a/src/internal/abi/map.go b/src/internal/abi/map.go deleted file mode 100644 index 12ad1b89..00000000 --- a/src/internal/abi/map.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package abi - -// Map constants common to several packages -// runtime/runtime-gdb.py:MapTypePrinter contains its own copy -const ( - // Maximum number of key/elem pairs a bucket can hold. - MapBucketCountBits = 3 // log2 of number of elements in a bucket. - MapBucketCount = 1 << MapBucketCountBits - - // Maximum key or elem size to keep inline (instead of mallocing per element). - // Must fit in a uint8. - // Note: fast map functions cannot handle big elems (bigger than MapMaxElemBytes). - MapMaxKeyBytes = 128 - MapMaxElemBytes = 128 // Must fit in a uint8. -) diff --git a/src/internal/abi/map_noswiss.go b/src/internal/abi/map_noswiss.go new file mode 100644 index 00000000..ff8609ef --- /dev/null +++ b/src/internal/abi/map_noswiss.go @@ -0,0 +1,54 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package abi + +import ( + "unsafe" +) + +// Map constants common to several packages +// runtime/runtime-gdb.py:MapTypePrinter contains its own copy +const ( + // Maximum number of key/elem pairs a bucket can hold. + OldMapBucketCountBits = 3 // log2 of number of elements in a bucket. + OldMapBucketCount = 1 << OldMapBucketCountBits + + // Maximum key or elem size to keep inline (instead of mallocing per element). + // Must fit in a uint8. + // Note: fast map functions cannot handle big elems (bigger than MapMaxElemBytes). + OldMapMaxKeyBytes = 128 + OldMapMaxElemBytes = 128 // Must fit in a uint8. +) + +type OldMapType struct { + Type + Key *Type + Elem *Type + Bucket *Type // internal type representing a hash bucket + // function for hashing keys (ptr to key, seed) -> hash + Hasher func(unsafe.Pointer, uintptr) uintptr + KeySize uint8 // size of key slot + ValueSize uint8 // size of elem slot + BucketSize uint16 // size of bucket + Flags uint32 +} + +// Note: flag values must match those used in the TMAP case +// in ../cmd/compile/internal/reflectdata/reflect.go:writeType. +func (mt *OldMapType) IndirectKey() bool { // store ptr to key instead of key itself + return mt.Flags&1 != 0 +} +func (mt *OldMapType) IndirectElem() bool { // store ptr to elem instead of elem itself + return mt.Flags&2 != 0 +} +func (mt *OldMapType) ReflexiveKey() bool { // true if k==k for all keys + return mt.Flags&4 != 0 +} +func (mt *OldMapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite + return mt.Flags&8 != 0 +} +func (mt *OldMapType) HashMightPanic() bool { // true if hash function might panic + return mt.Flags&16 != 0 +} diff --git a/src/internal/abi/map_select_noswiss.go b/src/internal/abi/map_select_noswiss.go new file mode 100644 index 00000000..ab2b69de --- /dev/null +++ b/src/internal/abi/map_select_noswiss.go @@ -0,0 +1,10 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +package abi + +// See comment in map_select_swiss.go. +type mapType = OldMapType diff --git a/src/internal/abi/map_select_swiss.go b/src/internal/abi/map_select_swiss.go new file mode 100644 index 00000000..88a0bb2e --- /dev/null +++ b/src/internal/abi/map_select_swiss.go @@ -0,0 +1,22 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package abi + +// Select the map type that this binary is built using. This is for common +// lookup methods like Type.Key to know which type to use. +// +// Note that mapType *must not be used by any functions called in the +// compiler to build a target program* because the compiler must use the map +// type determined by run-time GOEXPERIMENT, not the build tags used to build +// the compiler. +// +// TODO(prattmic): This package is rather confusing because it has many +// functions that can't be used by the compiler (e.g., Type.Uncommon depends on +// the layout of type + uncommon objects in the binary. It would be incorrect +// for an ad-hoc local Type object). It may be best to move code that isn't +// usable by the compiler out of the package. +type mapType = SwissMapType diff --git a/src/internal/abi/map_swiss.go b/src/internal/abi/map_swiss.go new file mode 100644 index 00000000..6c855667 --- /dev/null +++ b/src/internal/abi/map_swiss.go @@ -0,0 +1,64 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package abi + +import ( + "unsafe" +) + +// Map constants common to several packages +// runtime/runtime-gdb.py:MapTypePrinter contains its own copy +const ( + // Number of bits in the group.slot count. + SwissMapGroupSlotsBits = 3 + + // Number of slots in a group. + SwissMapGroupSlots = 1 << SwissMapGroupSlotsBits // 8 + + // Maximum key or elem size to keep inline (instead of mallocing per element). + // Must fit in a uint8. + SwissMapMaxKeyBytes = 128 + SwissMapMaxElemBytes = 128 + + ctrlEmpty = 0b10000000 + bitsetLSB = 0x0101010101010101 + + // Value of control word with all empty slots. + SwissMapCtrlEmpty = bitsetLSB * uint64(ctrlEmpty) +) + +type SwissMapType struct { + Type + Key *Type + Elem *Type + Group *Type // internal type representing a slot group + // function for hashing keys (ptr to key, seed) -> hash + Hasher func(unsafe.Pointer, uintptr) uintptr + GroupSize uintptr // == Group.Size_ + SlotSize uintptr // size of key/elem slot + ElemOff uintptr // offset of elem in key/elem slot + Flags uint32 +} + +// Flag values +const ( + SwissMapNeedKeyUpdate = 1 << iota + SwissMapHashMightPanic + SwissMapIndirectKey + SwissMapIndirectElem +) + +func (mt *SwissMapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite + return mt.Flags&SwissMapNeedKeyUpdate != 0 +} +func (mt *SwissMapType) HashMightPanic() bool { // true if hash function might panic + return mt.Flags&SwissMapHashMightPanic != 0 +} +func (mt *SwissMapType) IndirectKey() bool { // store ptr to key instead of key itself + return mt.Flags&SwissMapIndirectKey != 0 +} +func (mt *SwissMapType) IndirectElem() bool { // store ptr to elem instead of elem itself + return mt.Flags&SwissMapIndirectElem != 0 +} diff --git a/src/internal/abi/type.go b/src/internal/abi/type.go index b8eefe0d..1c1793fc 100644 --- a/src/internal/abi/type.go +++ b/src/internal/abi/type.go @@ -29,8 +29,16 @@ type Type struct { // (ptr to object A, ptr to object B) -> ==? Equal func(unsafe.Pointer, unsafe.Pointer) bool // GCData stores the GC type data for the garbage collector. - // If the KindGCProg bit is set in kind, GCData is a GC program. - // Otherwise it is a ptrmask bitmap. See mbitmap.go for details. + // Normally, GCData points to a bitmask that describes the + // ptr/nonptr fields of the type. The bitmask will have at + // least PtrBytes/ptrSize bits. + // If the TFlagGCMaskOnDemand bit is set, GCData is instead a + // **byte and the pointer to the bitmask is one dereference away. + // The runtime will build the bitmask if needed. + // (See runtime/type.go:getGCMask.) + // Note: multiple types may have the same value of GCData, + // including when TFlagGCMaskOnDemand is set. The types will, of course, + // have the same pointer layout (but not necessarily the same size). GCData *byte Str NameOff // string form PtrToThis TypeOff // type for pointer to this type, may be zero @@ -73,7 +81,6 @@ const ( const ( // TODO (khr, drchase) why aren't these in TFlag? Investigate, fix if possible. KindDirectIface Kind = 1 << 5 - KindGCProg Kind = 1 << 6 // Type.gc points to GC program KindMask Kind = (1 << 5) - 1 ) @@ -112,11 +119,12 @@ const ( // this type as a single region of t.size bytes. TFlagRegularMemory TFlag = 1 << 3 - // TFlagUnrolledBitmap marks special types that are unrolled-bitmap - // versions of types with GC programs. - // These types need to be deallocated when the underlying object - // is freed. - TFlagUnrolledBitmap TFlag = 1 << 4 + // TFlagGCMaskOnDemand means that the GC pointer bitmask will be + // computed on demand at runtime instead of being precomputed at + // compile time. If this flag is set, the GCData field effectively + // has type **byte instead of *byte. The runtime will store a + // pointer to the GC pointer bitmask in *GCData. + TFlagGCMaskOnDemand TFlag = 1 << 4 ) // NameOff is the offset to a name from moduledata.types. See resolveNameOff in runtime. @@ -206,6 +214,9 @@ func (t *Type) IsDirectIface() bool { } func (t *Type) GcSlice(begin, end uintptr) []byte { + if t.TFlag&TFlagGCMaskOnDemand != 0 { + panic("GcSlice can't handle on-demand gcdata types") + } return unsafe.Slice(t.GCData, int(end))[begin:] } @@ -350,7 +361,7 @@ func (t *Type) Uncommon() *UncommonType { return &(*u)(unsafe.Pointer(t)).u case Map: type u struct { - MapType + mapType u UncommonType } return &(*u)(unsafe.Pointer(t)).u @@ -379,7 +390,7 @@ func (t *Type) Elem() *Type { tt := (*ChanType)(unsafe.Pointer(t)) return tt.Elem case Map: - tt := (*MapType)(unsafe.Pointer(t)) + tt := (*mapType)(unsafe.Pointer(t)) return tt.Elem case Pointer: tt := (*PtrType)(unsafe.Pointer(t)) @@ -399,12 +410,12 @@ func (t *Type) StructType() *StructType { return (*StructType)(unsafe.Pointer(t)) } -// MapType returns t cast to a *MapType, or nil if its tag does not match. -func (t *Type) MapType() *MapType { +// MapType returns t cast to a *OldMapType or *SwissMapType, or nil if its tag does not match. +func (t *Type) MapType() *mapType { if t.Kind() != Map { return nil } - return (*MapType)(unsafe.Pointer(t)) + return (*mapType)(unsafe.Pointer(t)) } // ArrayType returns t cast to a *ArrayType, or nil if its tag does not match. @@ -464,40 +475,9 @@ func (t *Type) NumMethod() int { // NumMethod returns the number of interface methods in the type's method set. func (t *InterfaceType) NumMethod() int { return len(t.Methods) } -type MapType struct { - Type - Key *Type - Elem *Type - Bucket *Type // internal type representing a hash bucket - // function for hashing keys (ptr to key, seed) -> hash - Hasher func(unsafe.Pointer, uintptr) uintptr - KeySize uint8 // size of key slot - ValueSize uint8 // size of elem slot - BucketSize uint16 // size of bucket - Flags uint32 -} - -// Note: flag values must match those used in the TMAP case -// in ../cmd/compile/internal/reflectdata/reflect.go:writeType. -func (mt *MapType) IndirectKey() bool { // store ptr to key instead of key itself - return mt.Flags&1 != 0 -} -func (mt *MapType) IndirectElem() bool { // store ptr to elem instead of elem itself - return mt.Flags&2 != 0 -} -func (mt *MapType) ReflexiveKey() bool { // true if k==k for all keys - return mt.Flags&4 != 0 -} -func (mt *MapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite - return mt.Flags&8 != 0 -} -func (mt *MapType) HashMightPanic() bool { // true if hash function might panic - return mt.Flags&16 != 0 -} - func (t *Type) Key() *Type { if t.Kind() == Map { - return (*MapType)(unsafe.Pointer(t)).Key + return (*mapType)(unsafe.Pointer(t)).Key } return nil } diff --git a/src/internal/buildcfg/cfg.go b/src/internal/buildcfg/cfg.go index a16e76b3..fca09bf8 100644 --- a/src/internal/buildcfg/cfg.go +++ b/src/internal/buildcfg/cfg.go @@ -23,7 +23,7 @@ var ( GOROOT = os.Getenv("GOROOT") // cached for efficiency GOARCH = envOr("GOARCH", defaultGOARCH) GOOS = envOr("GOOS", defaultGOOS) - GO386 = envOr("GO386", defaultGO386) + GO386 = envOr("GO386", DefaultGO386) GOAMD64 = goamd64() GOARM = goarm() GOARM64 = goarm64() @@ -34,6 +34,7 @@ var ( GOWASM = gowasm() ToolTags = toolTags() GO_LDSO = defaultGO_LDSO + GOFIPS140 = gofips140() Version = version ) @@ -56,7 +57,7 @@ func envOr(key, value string) string { } func goamd64() int { - switch v := envOr("GOAMD64", defaultGOAMD64); v { + switch v := envOr("GOAMD64", DefaultGOAMD64); v { case "v1": return 1 case "v2": @@ -67,15 +68,56 @@ func goamd64() int { return 4 } Error = fmt.Errorf("invalid GOAMD64: must be v1, v2, v3, v4") - return int(defaultGOAMD64[len("v")] - '0') + return int(DefaultGOAMD64[len("v")] - '0') } -type goarmFeatures struct { +func gofips140() string { + v := envOr("GOFIPS140", DefaultGOFIPS140) + switch v { + case "off", "latest", "inprocess", "certified": + return v + } + if isFIPSVersion(v) { + return v + } + Error = fmt.Errorf("invalid GOFIPS140: must be off, latest, inprocess, certified, or vX.Y.Z") + return DefaultGOFIPS140 +} + +// isFIPSVersion reports whether v is a valid FIPS version, +// of the form vX.Y.Z. +func isFIPSVersion(v string) bool { + if !strings.HasPrefix(v, "v") { + return false + } + v, ok := skipNum(v[len("v"):]) + if !ok || !strings.HasPrefix(v, ".") { + return false + } + v, ok = skipNum(v[len("."):]) + if !ok || !strings.HasPrefix(v, ".") { + return false + } + v, ok = skipNum(v[len("."):]) + return ok && v == "" +} + +// skipNum skips the leading text matching [0-9]+ +// in s, returning the rest and whether such text was found. +func skipNum(s string) (rest string, ok bool) { + i := 0 + for i < len(s) && '0' <= s[i] && s[i] <= '9' { + i++ + } + return s[i:], i > 0 +} + +type GoarmFeatures struct { Version int SoftFloat bool } -func (g goarmFeatures) String() string { +func (g GoarmFeatures) String() string { armStr := strconv.Itoa(g.Version) if g.SoftFloat { armStr += ",softfloat" @@ -85,12 +127,12 @@ func (g goarmFeatures) String() string { return armStr } -func goarm() (g goarmFeatures) { +func goarm() (g GoarmFeatures) { const ( softFloatOpt = ",softfloat" hardFloatOpt = ",hardfloat" ) - def := defaultGOARM + def := DefaultGOARM if GOOS == "android" && GOARCH == "arm" { // Android arm devices always support GOARM=7. def = "7" @@ -186,14 +228,14 @@ func ParseGoarm64(v string) (g Goarm64Features, e error) { default: e = fmt.Errorf("invalid GOARM64: must start with v8.{0-9} or v9.{0-5} and may optionally end in %q and/or %q", lseOpt, cryptoOpt) - g.Version = defaultGOARM64 + g.Version = DefaultGOARM64 } return } func goarm64() (g Goarm64Features) { - g, Error = ParseGoarm64(envOr("GOARM64", defaultGOARM64)) + g, Error = ParseGoarm64(envOr("GOARM64", DefaultGOARM64)) return } @@ -229,25 +271,25 @@ func (g Goarm64Features) Supports(s string) bool { } func gomips() string { - switch v := envOr("GOMIPS", defaultGOMIPS); v { + switch v := envOr("GOMIPS", DefaultGOMIPS); v { case "hardfloat", "softfloat": return v } Error = fmt.Errorf("invalid GOMIPS: must be hardfloat, softfloat") - return defaultGOMIPS + return DefaultGOMIPS } func gomips64() string { - switch v := envOr("GOMIPS64", defaultGOMIPS64); v { + switch v := envOr("GOMIPS64", DefaultGOMIPS64); v { case "hardfloat", "softfloat": return v } Error = fmt.Errorf("invalid GOMIPS64: must be hardfloat, softfloat") - return defaultGOMIPS64 + return DefaultGOMIPS64 } func goppc64() int { - switch v := envOr("GOPPC64", defaultGOPPC64); v { + switch v := envOr("GOPPC64", DefaultGOPPC64); v { case "power8": return 8 case "power9": @@ -256,18 +298,18 @@ func goppc64() int { return 10 } Error = fmt.Errorf("invalid GOPPC64: must be power8, power9, power10") - return int(defaultGOPPC64[len("power")] - '0') + return int(DefaultGOPPC64[len("power")] - '0') } func goriscv64() int { - switch v := envOr("GORISCV64", defaultGORISCV64); v { + switch v := envOr("GORISCV64", DefaultGORISCV64); v { case "rva20u64": return 20 case "rva22u64": return 22 } Error = fmt.Errorf("invalid GORISCV64: must be rva20u64, rva22u64") - v := defaultGORISCV64[len("rva"):] + v := DefaultGORISCV64[len("rva"):] i := strings.IndexFunc(v, func(r rune) bool { return r < '0' || r > '9' }) diff --git a/src/internal/buildcfg/cfg_test.go b/src/internal/buildcfg/cfg_test.go index d01cdd01..757270b7 100644 --- a/src/internal/buildcfg/cfg_test.go +++ b/src/internal/buildcfg/cfg_test.go @@ -123,3 +123,39 @@ func TestGogoarchTags(t *testing.T) { GOARCH = old_goarch GOARM64 = old_goarm64 } + +var goodFIPS = []string{ + "v1.0.0", + "v1.0.1", + "v1.2.0", + "v1.2.3", +} + +var badFIPS = []string{ + "v1.0.0-fips", + "v1.0.0+fips", + "1.0.0", + "x1.0.0", +} + +func TestIsFIPSVersion(t *testing.T) { + // good + for _, s := range goodFIPS { + if !isFIPSVersion(s) { + t.Errorf("isFIPSVersion(%q) = false, want true", s) + } + } + // truncated + const v = "v1.2.3" + for i := 0; i < len(v); i++ { + if isFIPSVersion(v[:i]) { + t.Errorf("isFIPSVersion(%q) = true, want false", v[:i]) + } + } + // bad + for _, s := range badFIPS { + if isFIPSVersion(s) { + t.Errorf("isFIPSVersion(%q) = true, want false", s) + } + } +} diff --git a/src/internal/buildcfg/exp.go b/src/internal/buildcfg/exp.go index 7c7cefba..332c9afa 100644 --- a/src/internal/buildcfg/exp.go +++ b/src/internal/buildcfg/exp.go @@ -67,10 +67,20 @@ func ParseGOEXPERIMENT(goos, goarch, goexp string) (*ExperimentFlags, error) { regabiSupported = true } + var haveXchg8 bool + switch goarch { + case "386", "amd64", "arm", "arm64", "ppc64le", "ppc64": + haveXchg8 = true + } + baseline := goexperiment.Flags{ RegabiWrappers: regabiSupported, RegabiArgs: regabiSupported, CoverageRedesign: true, + AliasTypeParams: true, + SwissMap: true, + SpinbitMutex: haveXchg8, + SyncHashTrieMap: true, } // Start with the statically enabled set of experiments. diff --git a/src/internal/bytealg/compare_loong64.s b/src/internal/bytealg/compare_loong64.s index df72a112..99c8cda7 100644 --- a/src/internal/bytealg/compare_loong64.s +++ b/src/internal/bytealg/compare_loong64.s @@ -28,58 +28,136 @@ TEXT runtime·cmpstring(SB),NOSPLIT,$0-40 // R7 length of b // R4 points to the start of a // R6 points to the start of b -// R13 points to the return value (-1/0/1) +// for regabi the return value (-1/0/1) in R4 TEXT cmpbody<>(SB),NOSPLIT|NOFRAME,$0 - BEQ R4, R6, samebytes // same start of a and b + BEQ R4, R6, cmp_len // same start of a and b, then compare lengths SGTU R5, R7, R9 - BNE R0, R9, r2_lt_r1 + BNE R9, b_lt_a MOVV R5, R14 JMP entry -r2_lt_r1: - MOVV R7, R14 // R14 is min(R4, R5) +b_lt_a: + MOVV R7, R14 // R14 is min(R5, R7) entry: - ADDV R4, R14, R12 // R6 start of a, R14 end of a - BEQ R4, R12, samebytes // length is 0 + ADDV R4, R14, R12 // R4 start of a, R12 end of a + BEQ R4, R12, cmp_len // minlength is 0 - SRLV $4, R14 // R14 is number of chunks - BEQ R0, R14, byte_loop +tail: + MOVV $2, R15 + BLT R14, R15, cmp1 // min < 2 + SLLV $1, R15 + BLT R14, R15, cmp2 // min < 4 + SLLV $1, R15 + BLT R14, R15, cmp4 // min < 8 + SLLV $1, R15 + BLT R14, R15, cmp8 // min < 16 + SLLV $1, R15 + BLT R14, R15, cmp16 // min < 32 - // make sure both a and b are aligned. - OR R4, R6, R15 - AND $7, R15 - BNE R0, R15, byte_loop - - PCALIGN $16 -chunk16_loop: - BEQ R0, R14, byte_loop +// When min >= 32 bytes, enter the cmp32_loop loop processing: +// take out 4 8-bytes from a and b in turn for comparison. +cmp32_loop: MOVV (R4), R8 MOVV (R6), R9 - BNE R8, R9, byte_loop - MOVV 8(R4), R16 - MOVV 8(R6), R17 + MOVV 8(R4), R10 + MOVV 8(R6), R11 + BNE R8, R9, cmp8a + BNE R10, R11, cmp8b + MOVV 16(R4), R8 + MOVV 16(R6), R9 + MOVV 24(R4), R10 + MOVV 24(R6), R11 + BNE R8, R9, cmp8a + BNE R10, R11, cmp8b + ADDV $32, R4 + ADDV $32, R6 + SUBV $32, R14 + BGE R14, R15, cmp32_loop + BEQ R14, cmp_len + +check16: + MOVV $16, R15 + BLT R14, R15, check8 +cmp16: + MOVV (R4), R8 + MOVV (R6), R9 + MOVV 8(R4), R10 + MOVV 8(R6), R11 + BNE R8, R9, cmp8a + BNE R10, R11, cmp8b ADDV $16, R4 ADDV $16, R6 - SUBVU $1, R14 - BEQ R16, R17, chunk16_loop - SUBV $8, R4 - SUBV $8, R6 + SUBV $16, R14 + BEQ R14, cmp_len -byte_loop: - BEQ R4, R12, samebytes +check8: + MOVV $8, R15 + BLT R14, R15, check4 +cmp8: + MOVV (R4), R8 + MOVV (R6), R9 + BNE R8, R9, cmp8a + ADDV $8, R4 + ADDV $8, R6 + SUBV $8, R14 + BEQ R14, cmp_len + +check4: + MOVV $4, R15 + BLT R14, R15, check2 +cmp4: + MOVW (R4), R8 + MOVW (R6), R9 + BNE R8, R9, cmp8a + ADDV $4, R4 + ADDV $4, R6 + SUBV $4, R14 + BEQ R14, cmp_len + +check2: + MOVV $2, R15 + BLT R14, R15, cmp1 +cmp2: + MOVH (R4), R8 + MOVH (R6), R9 + BNE R8, R9, cmp8a + ADDV $2, R4 + ADDV $2, R6 + SUBV $2, R14 + BEQ R14, cmp_len + +cmp1: + BEQ R14, cmp_len MOVBU (R4), R8 - ADDVU $1, R4 MOVBU (R6), R9 - ADDVU $1, R6 - BEQ R8, R9, byte_loop + BNE R8, R9, byte_cmp + JMP cmp_len + // Compare 8/4/2 bytes taken from R8/R9 that are known to differ. +cmp8a: + MOVV R8, R10 + MOVV R9, R11 + + // Compare 8/4/2 bytes taken from R10/R11 that are known to differ. +cmp8b: + MOVV $0xff, R15 + + // Take single bytes from R10/R11 in turn for cyclic comparison. +cmp8_loop: + AND R10, R15, R8 + AND R11, R15, R9 + BNE R8, R9, byte_cmp + SLLV $8, R15 + JMP cmp8_loop + + // Compare 1 bytes taken from R8/R9 that are known to differ. byte_cmp: - SGTU R8, R9, R4 // R12 = 1 if (R8 > R9) + SGTU R8, R9, R4 // R4 = 1 if (R8 > R9) BNE R0, R4, ret MOVV $-1, R4 JMP ret -samebytes: +cmp_len: SGTU R5, R7, R8 SGTU R7, R5, R9 SUBV R9, R8, R4 diff --git a/src/internal/bytealg/compare_ppc64x.s b/src/internal/bytealg/compare_ppc64x.s index 2629251e..a3d56cfd 100644 --- a/src/internal/bytealg/compare_ppc64x.s +++ b/src/internal/bytealg/compare_ppc64x.s @@ -61,7 +61,7 @@ TEXT ·Compare(SB),NOSPLIT|NOFRAME,$0-56 CMP R3,R6,CR7 ISEL CR0LT,R4,R7,R9 SETB_CR0(R3) - BC $12,30,LR // beqlr cr7 + BEQ CR7,LR BR cmpbody<>(SB) TEXT runtime·cmpstring(SB),NOSPLIT|NOFRAME,$0-40 @@ -83,7 +83,7 @@ TEXT runtime·cmpstring(SB),NOSPLIT|NOFRAME,$0-40 MOVD R5,R6 MOVD R3,R5 SETB_CR0(R3) - BC $12,30,LR // beqlr cr7 + BEQ CR7,LR BR cmpbody<>(SB) #ifdef GOARCH_ppc64le @@ -143,7 +143,7 @@ cmp64_loop: ADD $64,R5,R5 // increment to next 64 bytes of A ADD $64,R6,R6 // increment to next 64 bytes of B BDNZ cmp64_loop - BC $12,2,LR // beqlr + BEQ CR0,LR // beqlr // Finish out tail with minimal overlapped checking. // Note, 0 tail is handled by beqlr above. @@ -215,7 +215,7 @@ cmp32: // 32 - 63B VCMPEQUDCC V3,V4,V1 BGE CR6,different - BC $12,2,LR // beqlr + BEQ CR0,LR ADD R9,R10,R10 LXVD2X (R9)(R5),V3 @@ -236,7 +236,7 @@ cmp16: // 16 - 31B LXVD2X (R0)(R6),V4 VCMPEQUDCC V3,V4,V1 BGE CR6,different - BC $12,2,LR // beqlr + BEQ CR0,LR LXVD2X (R9)(R5),V3 LXVD2X (R9)(R6),V4 diff --git a/src/internal/bytealg/equal_arm64.s b/src/internal/bytealg/equal_arm64.s index 4db95154..408ab374 100644 --- a/src/internal/bytealg/equal_arm64.s +++ b/src/internal/bytealg/equal_arm64.s @@ -5,25 +5,11 @@ #include "go_asm.h" #include "textflag.h" -// memequal(a, b unsafe.Pointer, size uintptr) bool -TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 - // short path to handle 0-byte case - CBZ R2, equal - // short path to handle equal pointers - CMP R0, R1 - BEQ equal - B memeqbody<>(SB) -equal: - MOVD $1, R0 - RET - // memequal_varlen(a, b unsafe.Pointer) bool TEXT runtime·memequal_varlen(SB),NOSPLIT,$0-17 - CMP R0, R1 - BEQ eq MOVD 8(R26), R2 // compiler stores size at offset 8 in the closure CBZ R2, eq - B memeqbody<>(SB) + B runtime·memequal(SB) eq: MOVD $1, R0 RET @@ -33,7 +19,13 @@ eq: // R1: pointer b // R2: data len // at return: result in R0 -TEXT memeqbody<>(SB),NOSPLIT,$0 +// memequal(a, b unsafe.Pointer, size uintptr) bool +TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 + // short path to handle 0-byte case + CBZ R2, equal + // short path to handle equal pointers + CMP R0, R1 + BEQ equal CMP $1, R2 // handle 1-byte special case for better performance BEQ one @@ -91,6 +83,7 @@ tail: EOR R4, R5 CBNZ R5, not_equal B equal + PCALIGN $16 lt_8: TBZ $2, R2, lt_4 MOVWU (R0), R4 @@ -103,6 +96,7 @@ lt_8: EOR R4, R5 CBNZ R5, not_equal B equal + PCALIGN $16 lt_4: TBZ $1, R2, lt_2 MOVHU.P 2(R0), R4 diff --git a/src/internal/bytealg/indexbyte_riscv64.s b/src/internal/bytealg/indexbyte_riscv64.s index de00983c..fde00da0 100644 --- a/src/internal/bytealg/indexbyte_riscv64.s +++ b/src/internal/bytealg/indexbyte_riscv64.s @@ -10,33 +10,14 @@ TEXT ·IndexByte(SB),NOSPLIT,$0-40 // X11 = b_len // X12 = b_cap (unused) // X13 = byte to find - AND $0xff, X13 - MOV X10, X12 // store base for later - ADD X10, X11 // end - SUB $1, X10 - -loop: - ADD $1, X10 - BEQ X10, X11, notfound - MOVBU (X10), X14 - BNE X13, X14, loop - - SUB X12, X10 // remove base - RET - -notfound: - MOV $-1, X10 - RET - -TEXT ·IndexByteString(SB),NOSPLIT,$0-32 - // X10 = b_base - // X11 = b_len - // X12 = byte to find - AND $0xff, X12 + AND $0xff, X13, X12 // x12 byte to look for MOV X10, X13 // store base for later - ADD X10, X11 // end - SUB $1, X10 + SLTI $24, X11, X14 + ADD X10, X11 // end + BEQZ X14, bigBody + + SUB $1, X10 loop: ADD $1, X10 BEQ X10, X11, notfound @@ -49,3 +30,110 @@ loop: notfound: MOV $-1, X10 RET + +bigBody: + JMP indexByteBig<>(SB) + +TEXT ·IndexByteString(SB),NOSPLIT,$0-32 + // X10 = b_base + // X11 = b_len + // X12 = byte to find + + AND $0xff, X12 // x12 byte to look for + MOV X10, X13 // store base for later + + SLTI $24, X11, X14 + ADD X10, X11 // end + BEQZ X14, bigBody + + SUB $1, X10 +loop: + ADD $1, X10 + BEQ X10, X11, notfound + MOVBU (X10), X14 + BNE X12, X14, loop + + SUB X13, X10 // remove base + RET + +notfound: + MOV $-1, X10 + RET + +bigBody: + JMP indexByteBig<>(SB) + +TEXT indexByteBig<>(SB),NOSPLIT|NOFRAME,$0 + // On entry + // X10 = b_base + // X11 = end + // X12 = byte to find + // X13 = b_base + // X11 is at least 16 bytes > X10 + + // On exit + // X10 = index of first instance of sought byte, if found, or -1 otherwise + + // Process the first few bytes until we get to an 8 byte boundary + // No need to check for end here as we have at least 16 bytes in + // the buffer. + +unalignedloop: + AND $7, X10, X14 + BEQZ X14, aligned + MOVBU (X10), X14 + BEQ X12, X14, found + ADD $1, X10 + JMP unalignedloop + +aligned: + AND $~7, X11, X15 // X15 = end of aligned data + + // We have at least 9 bytes left + + // Use 'Determine if a word has a byte equal to n' bit hack from + // https://graphics.stanford.edu/~seander/bithacks.html to determine + // whether the byte is present somewhere in the next 8 bytes of the + // array. + + MOV $0x0101010101010101, X16 + SLLI $7, X16, X17 // X17 = 0x8080808080808080 + + MUL X12, X16, X18 // broadcast X12 to every byte in X18 + +alignedloop: + MOV (X10), X14 + XOR X14, X18, X19 + + // If the LSB in X12 is present somewhere in the 8 bytes we've just + // loaded into X14 then at least one of the bytes in X19 will be 0 + // after the XOR. If any of the bytes in X19 are zero then + // + // ((X19 - X16) & (~X19) & X17) + // + // will be non-zero. The expression will evaluate to zero if none of + // the bytes in X19 are zero, i.e., X12 is not present in X14. + + SUB X16, X19, X20 + ANDN X19, X17, X21 + AND X20, X21 + BNEZ X21, tailloop // If X21 != 0 X12 is present in X14 + ADD $8, X10 + BNE X10, X15, alignedloop + +tailloop: + SUB $1, X10 + +loop: + ADD $1, X10 + BEQ X10, X11, notfound + MOVBU (X10), X14 + BNE X12, X14, loop + +found: + SUB X13, X10 // remove base + RET + +notfound: + MOV $-1, X10 + RET diff --git a/src/internal/byteorder/byteorder.go b/src/internal/byteorder/byteorder.go index ba37856c..01500a87 100644 --- a/src/internal/byteorder/byteorder.go +++ b/src/internal/byteorder/byteorder.go @@ -6,30 +6,30 @@ // little and big endian integer types from/to byte slices. package byteorder -func LeUint16(b []byte) uint16 { +func LEUint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[0]) | uint16(b[1])<<8 } -func LePutUint16(b []byte, v uint16) { +func LEPutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) } -func LeAppendUint16(b []byte, v uint16) []byte { +func LEAppendUint16(b []byte, v uint16) []byte { return append(b, byte(v), byte(v>>8), ) } -func LeUint32(b []byte) uint32 { +func LEUint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 } -func LePutUint32(b []byte, v uint32) { +func LEPutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) @@ -37,7 +37,7 @@ func LePutUint32(b []byte, v uint32) { b[3] = byte(v >> 24) } -func LeAppendUint32(b []byte, v uint32) []byte { +func LEAppendUint32(b []byte, v uint32) []byte { return append(b, byte(v), byte(v>>8), @@ -46,13 +46,13 @@ func LeAppendUint32(b []byte, v uint32) []byte { ) } -func LeUint64(b []byte) uint64 { +func LEUint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 } -func LePutUint64(b []byte, v uint64) { +func LEPutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) @@ -64,7 +64,7 @@ func LePutUint64(b []byte, v uint64) { b[7] = byte(v >> 56) } -func LeAppendUint64(b []byte, v uint64) []byte { +func LEAppendUint64(b []byte, v uint64) []byte { return append(b, byte(v), byte(v>>8), @@ -77,30 +77,30 @@ func LeAppendUint64(b []byte, v uint64) []byte { ) } -func BeUint16(b []byte) uint16 { +func BEUint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[1]) | uint16(b[0])<<8 } -func BePutUint16(b []byte, v uint16) { +func BEPutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 8) b[1] = byte(v) } -func BeAppendUint16(b []byte, v uint16) []byte { +func BEAppendUint16(b []byte, v uint16) []byte { return append(b, byte(v>>8), byte(v), ) } -func BeUint32(b []byte) uint32 { +func BEUint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 } -func BePutUint32(b []byte, v uint32) { +func BEPutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 24) b[1] = byte(v >> 16) @@ -108,7 +108,7 @@ func BePutUint32(b []byte, v uint32) { b[3] = byte(v) } -func BeAppendUint32(b []byte, v uint32) []byte { +func BEAppendUint32(b []byte, v uint32) []byte { return append(b, byte(v>>24), byte(v>>16), @@ -117,13 +117,13 @@ func BeAppendUint32(b []byte, v uint32) []byte { ) } -func BeUint64(b []byte) uint64 { +func BEUint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } -func BePutUint64(b []byte, v uint64) { +func BEPutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 56) b[1] = byte(v >> 48) @@ -135,7 +135,7 @@ func BePutUint64(b []byte, v uint64) { b[7] = byte(v) } -func BeAppendUint64(b []byte, v uint64) []byte { +func BEAppendUint64(b []byte, v uint64) []byte { return append(b, byte(v>>56), byte(v>>48), diff --git a/src/internal/cfg/cfg.go b/src/internal/cfg/cfg.go index 08d210b7..93297697 100644 --- a/src/internal/cfg/cfg.go +++ b/src/internal/cfg/cfg.go @@ -37,12 +37,14 @@ const KnownEnv = ` GOARCH GOARM GOARM64 + GOAUTH GOBIN GOCACHE GOCACHEPROG GOENV GOEXE GOEXPERIMENT + GOFIPS140 GOFLAGS GOGCCFLAGS GOHOSTARCH diff --git a/src/internal/chacha8rand/chacha8.go b/src/internal/chacha8rand/chacha8.go index 8f1b4e53..96fb8726 100644 --- a/src/internal/chacha8rand/chacha8.go +++ b/src/internal/chacha8rand/chacha8.go @@ -53,10 +53,10 @@ func (s *State) Next() (uint64, bool) { // Init seeds the State with the given seed value. func (s *State) Init(seed [32]byte) { s.Init64([4]uint64{ - byteorder.LeUint64(seed[0*8:]), - byteorder.LeUint64(seed[1*8:]), - byteorder.LeUint64(seed[2*8:]), - byteorder.LeUint64(seed[3*8:]), + byteorder.LEUint64(seed[0*8:]), + byteorder.LEUint64(seed[1*8:]), + byteorder.LEUint64(seed[2*8:]), + byteorder.LEUint64(seed[3*8:]), }) } @@ -124,9 +124,9 @@ func Marshal(s *State) []byte { data := make([]byte, 6*8) copy(data, "chacha8:") used := (s.c/ctrInc)*chunk + s.i - byteorder.BePutUint64(data[1*8:], uint64(used)) + byteorder.BEPutUint64(data[1*8:], uint64(used)) for i, seed := range s.seed { - byteorder.LePutUint64(data[(2+i)*8:], seed) + byteorder.LEPutUint64(data[(2+i)*8:], seed) } return data } @@ -142,12 +142,12 @@ func Unmarshal(s *State, data []byte) error { if len(data) != 6*8 || string(data[:8]) != "chacha8:" { return new(errUnmarshalChaCha8) } - used := byteorder.BeUint64(data[1*8:]) + used := byteorder.BEUint64(data[1*8:]) if used > (ctrMax/ctrInc)*chunk-reseed { return new(errUnmarshalChaCha8) } for i := range s.seed { - s.seed[i] = byteorder.LeUint64(data[(2+i)*8:]) + s.seed[i] = byteorder.LEUint64(data[(2+i)*8:]) } s.c = ctrInc * (uint32(used) / chunk) block(&s.seed, &s.buf, s.c) diff --git a/src/internal/concurrent/hashtriemap.go b/src/internal/concurrent/hashtriemap.go deleted file mode 100644 index 4f7e730d..00000000 --- a/src/internal/concurrent/hashtriemap.go +++ /dev/null @@ -1,408 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package concurrent - -import ( - "internal/abi" - "internal/goarch" - "math/rand/v2" - "sync" - "sync/atomic" - "unsafe" -) - -// HashTrieMap is an implementation of a concurrent hash-trie. The implementation -// is designed around frequent loads, but offers decent performance for stores -// and deletes as well, especially if the map is larger. It's primary use-case is -// the unique package, but can be used elsewhere as well. -type HashTrieMap[K, V comparable] struct { - root *indirect[K, V] - keyHash hashFunc - keyEqual equalFunc - valEqual equalFunc - seed uintptr -} - -// NewHashTrieMap creates a new HashTrieMap for the provided key and value. -func NewHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { - var m map[K]V - mapType := abi.TypeOf(m).MapType() - ht := &HashTrieMap[K, V]{ - root: newIndirectNode[K, V](nil), - keyHash: mapType.Hasher, - keyEqual: mapType.Key.Equal, - valEqual: mapType.Elem.Equal, - seed: uintptr(rand.Uint64()), - } - return ht -} - -type hashFunc func(unsafe.Pointer, uintptr) uintptr -type equalFunc func(unsafe.Pointer, unsafe.Pointer) bool - -// Load returns the value stored in the map for a key, or nil if no -// value is present. -// The ok result indicates whether value was found in the map. -func (ht *HashTrieMap[K, V]) Load(key K) (value V, ok bool) { - hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) - - i := ht.root - hashShift := 8 * goarch.PtrSize - for hashShift != 0 { - hashShift -= nChildrenLog2 - - n := i.children[(hash>>hashShift)&nChildrenMask].Load() - if n == nil { - return *new(V), false - } - if n.isEntry { - return n.entry().lookup(key, ht.keyEqual) - } - i = n.indirect() - } - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") -} - -// LoadOrStore returns the existing value for the key if present. -// Otherwise, it stores and returns the given value. -// The loaded result is true if the value was loaded, false if stored. -func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) { - hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) - var i *indirect[K, V] - var hashShift uint - var slot *atomic.Pointer[node[K, V]] - var n *node[K, V] - for { - // Find the key or a candidate location for insertion. - i = ht.root - hashShift = 8 * goarch.PtrSize - haveInsertPoint := false - for hashShift != 0 { - hashShift -= nChildrenLog2 - - slot = &i.children[(hash>>hashShift)&nChildrenMask] - n = slot.Load() - if n == nil { - // We found a nil slot which is a candidate for insertion. - haveInsertPoint = true - break - } - if n.isEntry { - // We found an existing entry, which is as far as we can go. - // If it stays this way, we'll have to replace it with an - // indirect node. - if v, ok := n.entry().lookup(key, ht.keyEqual); ok { - return v, true - } - haveInsertPoint = true - break - } - i = n.indirect() - } - if !haveInsertPoint { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") - } - - // Grab the lock and double-check what we saw. - i.mu.Lock() - n = slot.Load() - if (n == nil || n.isEntry) && !i.dead.Load() { - // What we saw is still true, so we can continue with the insert. - break - } - // We have to start over. - i.mu.Unlock() - } - // N.B. This lock is held from when we broke out of the outer loop above. - // We specifically break this out so that we can use defer here safely. - // One option is to break this out into a new function instead, but - // there's so much local iteration state used below that this turns out - // to be cleaner. - defer i.mu.Unlock() - - var oldEntry *entry[K, V] - if n != nil { - oldEntry = n.entry() - if v, ok := oldEntry.lookup(key, ht.keyEqual); ok { - // Easy case: by loading again, it turns out exactly what we wanted is here! - return v, true - } - } - newEntry := newEntryNode(key, value) - if oldEntry == nil { - // Easy case: create a new entry and store it. - slot.Store(&newEntry.node) - } else { - // We possibly need to expand the entry already there into one or more new nodes. - // - // Publish the node last, which will make both oldEntry and newEntry visible. We - // don't want readers to be able to observe that oldEntry isn't in the tree. - slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i)) - } - return value, false -} - -// expand takes oldEntry and newEntry whose hashes conflict from bit 64 down to hashShift and -// produces a subtree of indirect nodes to hold the two new entries. -func (ht *HashTrieMap[K, V]) expand(oldEntry, newEntry *entry[K, V], newHash uintptr, hashShift uint, parent *indirect[K, V]) *node[K, V] { - // Check for a hash collision. - oldHash := ht.keyHash(unsafe.Pointer(&oldEntry.key), ht.seed) - if oldHash == newHash { - // Store the old entry in the new entry's overflow list, then store - // the new entry. - newEntry.overflow.Store(oldEntry) - return &newEntry.node - } - // We have to add an indirect node. Worse still, we may need to add more than one. - newIndirect := newIndirectNode(parent) - top := newIndirect - for { - if hashShift == 0 { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while inserting") - } - hashShift -= nChildrenLog2 // hashShift is for the level parent is at. We need to go deeper. - oi := (oldHash >> hashShift) & nChildrenMask - ni := (newHash >> hashShift) & nChildrenMask - if oi != ni { - newIndirect.children[oi].Store(&oldEntry.node) - newIndirect.children[ni].Store(&newEntry.node) - break - } - nextIndirect := newIndirectNode(newIndirect) - newIndirect.children[oi].Store(&nextIndirect.node) - newIndirect = nextIndirect - } - return &top.node -} - -// CompareAndDelete deletes the entry for key if its value is equal to old. -// -// If there is no current value for key in the map, CompareAndDelete returns false -// (even if the old value is the nil interface value). -func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { - hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) - var i *indirect[K, V] - var hashShift uint - var slot *atomic.Pointer[node[K, V]] - var n *node[K, V] - for { - // Find the key or return when there's nothing to delete. - i = ht.root - hashShift = 8 * goarch.PtrSize - found := false - for hashShift != 0 { - hashShift -= nChildrenLog2 - - slot = &i.children[(hash>>hashShift)&nChildrenMask] - n = slot.Load() - if n == nil { - // Nothing to delete. Give up. - return - } - if n.isEntry { - // We found an entry. Check if it matches. - if _, ok := n.entry().lookup(key, ht.keyEqual); !ok { - // No match, nothing to delete. - return - } - // We've got something to delete. - found = true - break - } - i = n.indirect() - } - if !found { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") - } - - // Grab the lock and double-check what we saw. - i.mu.Lock() - n = slot.Load() - if !i.dead.Load() { - if n == nil { - // Valid node that doesn't contain what we need. Nothing to delete. - i.mu.Unlock() - return - } - if n.isEntry { - // What we saw is still true, so we can continue with the delete. - break - } - } - // We have to start over. - i.mu.Unlock() - } - // Try to delete the entry. - e, deleted := n.entry().compareAndDelete(key, old, ht.keyEqual, ht.valEqual) - if !deleted { - // Nothing was actually deleted, which means the node is no longer there. - i.mu.Unlock() - return false - } - if e != nil { - // We didn't actually delete the whole entry, just one entry in the chain. - // Nothing else to do, since the parent is definitely not empty. - slot.Store(&e.node) - i.mu.Unlock() - return true - } - // Delete the entry. - slot.Store(nil) - - // Check if the node is now empty (and isn't the root), and delete it if able. - for i.parent != nil && i.empty() { - if hashShift == 8*goarch.PtrSize { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") - } - hashShift += nChildrenLog2 - - // Delete the current node in the parent. - parent := i.parent - parent.mu.Lock() - i.dead.Store(true) - parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) - i.mu.Unlock() - i = parent - } - i.mu.Unlock() - return true -} - -// All returns an iter.Seq2 that produces all key-value pairs in the map. -// The enumeration does not represent any consistent snapshot of the map, -// but is guaranteed to visit each unique key-value pair only once. It is -// safe to operate on the tree during iteration. No particular enumeration -// order is guaranteed. -func (ht *HashTrieMap[K, V]) All() func(yield func(K, V) bool) { - return func(yield func(key K, value V) bool) { - ht.iter(ht.root, yield) - } -} - -func (ht *HashTrieMap[K, V]) iter(i *indirect[K, V], yield func(key K, value V) bool) bool { - for j := range i.children { - n := i.children[j].Load() - if n == nil { - continue - } - if !n.isEntry { - if !ht.iter(n.indirect(), yield) { - return false - } - continue - } - e := n.entry() - for e != nil { - if !yield(e.key, e.value) { - return false - } - e = e.overflow.Load() - } - } - return true -} - -const ( - // 16 children. This seems to be the sweet spot for - // load performance: any smaller and we lose out on - // 50% or more in CPU performance. Any larger and the - // returns are minuscule (~1% improvement for 32 children). - nChildrenLog2 = 4 - nChildren = 1 << nChildrenLog2 - nChildrenMask = nChildren - 1 -) - -// indirect is an internal node in the hash-trie. -type indirect[K, V comparable] struct { - node[K, V] - dead atomic.Bool - mu sync.Mutex // Protects mutation to children and any children that are entry nodes. - parent *indirect[K, V] - children [nChildren]atomic.Pointer[node[K, V]] -} - -func newIndirectNode[K, V comparable](parent *indirect[K, V]) *indirect[K, V] { - return &indirect[K, V]{node: node[K, V]{isEntry: false}, parent: parent} -} - -func (i *indirect[K, V]) empty() bool { - nc := 0 - for j := range i.children { - if i.children[j].Load() != nil { - nc++ - } - } - return nc == 0 -} - -// entry is a leaf node in the hash-trie. -type entry[K, V comparable] struct { - node[K, V] - overflow atomic.Pointer[entry[K, V]] // Overflow for hash collisions. - key K - value V -} - -func newEntryNode[K, V comparable](key K, value V) *entry[K, V] { - return &entry[K, V]{ - node: node[K, V]{isEntry: true}, - key: key, - value: value, - } -} - -func (e *entry[K, V]) lookup(key K, equal equalFunc) (V, bool) { - for e != nil { - if equal(unsafe.Pointer(&e.key), abi.NoEscape(unsafe.Pointer(&key))) { - return e.value, true - } - e = e.overflow.Load() - } - return *new(V), false -} - -// compareAndDelete deletes an entry in the overflow chain if both the key and value compare -// equal. Returns the new entry chain and whether or not anything was deleted. -// -// compareAndDelete must be called under the mutex of the indirect node which e is a child of. -func (head *entry[K, V]) compareAndDelete(key K, value V, keyEqual, valEqual equalFunc) (*entry[K, V], bool) { - if keyEqual(unsafe.Pointer(&head.key), abi.NoEscape(unsafe.Pointer(&key))) && - valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&value))) { - // Drop the head of the list. - return head.overflow.Load(), true - } - i := &head.overflow - e := i.Load() - for e != nil { - if keyEqual(unsafe.Pointer(&e.key), abi.NoEscape(unsafe.Pointer(&key))) && - valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value))) { - i.Store(e.overflow.Load()) - return head, true - } - i = &e.overflow - e = e.overflow.Load() - } - return head, false -} - -// node is the header for a node. It's polymorphic and -// is actually either an entry or an indirect. -type node[K, V comparable] struct { - isEntry bool -} - -func (n *node[K, V]) entry() *entry[K, V] { - if !n.isEntry { - panic("called entry on non-entry node") - } - return (*entry[K, V])(unsafe.Pointer(n)) -} - -func (n *node[K, V]) indirect() *indirect[K, V] { - if n.isEntry { - panic("called indirect on entry node") - } - return (*indirect[K, V])(unsafe.Pointer(n)) -} diff --git a/src/internal/concurrent/hashtriemap_test.go b/src/internal/concurrent/hashtriemap_test.go deleted file mode 100644 index e233824c..00000000 --- a/src/internal/concurrent/hashtriemap_test.go +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package concurrent - -import ( - "fmt" - "math" - "runtime" - "strconv" - "strings" - "sync" - "testing" - "unsafe" -) - -func TestHashTrieMap(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - return NewHashTrieMap[string, int]() - }) -} - -func TestHashTrieMapBadHash(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - // Stub out the good hash function with a terrible one. - // Everything should still work as expected. - m := NewHashTrieMap[string, int]() - m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr { - return 0 - } - return m - }) -} - -func testHashTrieMap(t *testing.T, newMap func() *HashTrieMap[string, int]) { - t.Run("LoadEmpty", func(t *testing.T) { - m := newMap() - - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - }) - t.Run("LoadOrStore", func(t *testing.T) { - m := newMap() - - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - for i, s := range testData { - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - }) - t.Run("CompareAndDeleteAll", func(t *testing.T) { - m := newMap() - - for range 3 { - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - for i, s := range testData { - expectPresent(t, s, i)(m.Load(s)) - expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) - expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) - expectNotDeleted(t, s, i)(m.CompareAndDelete(s, i)) - expectMissing(t, s, 0)(m.Load(s)) - } - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - } - }) - t.Run("CompareAndDeleteOne", func(t *testing.T) { - m := newMap() - - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - expectNotDeleted(t, testData[15], math.MaxInt)(m.CompareAndDelete(testData[15], math.MaxInt)) - expectDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) - expectNotDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) - for i, s := range testData { - if i == 15 { - expectMissing(t, s, 0)(m.Load(s)) - } else { - expectPresent(t, s, i)(m.Load(s)) - } - } - }) - t.Run("DeleteMultiple", func(t *testing.T) { - m := newMap() - - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - for _, i := range []int{1, 105, 6, 85} { - expectNotDeleted(t, testData[i], math.MaxInt)(m.CompareAndDelete(testData[i], math.MaxInt)) - expectDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) - expectNotDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) - } - for i, s := range testData { - if i == 1 || i == 105 || i == 6 || i == 85 { - expectMissing(t, s, 0)(m.Load(s)) - } else { - expectPresent(t, s, i)(m.Load(s)) - } - } - }) - t.Run("All", func(t *testing.T) { - m := newMap() - - testAll(t, m, testDataMap(testData[:]), func(_ string, _ int) bool { - return true - }) - }) - t.Run("AllDelete", func(t *testing.T) { - m := newMap() - - testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { - expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) - return true - }) - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - }) - t.Run("ConcurrentLifecycleUnsharedKeys", func(t *testing.T) { - m := newMap() - - gmp := runtime.GOMAXPROCS(-1) - var wg sync.WaitGroup - for i := range gmp { - wg.Add(1) - go func(id int) { - defer wg.Done() - - makeKey := func(s string) string { - return s + "-" + strconv.Itoa(id) - } - for _, s := range testData { - key := makeKey(s) - expectMissing(t, key, 0)(m.Load(key)) - expectStored(t, key, id)(m.LoadOrStore(key, id)) - expectPresent(t, key, id)(m.Load(key)) - expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) - } - for _, s := range testData { - key := makeKey(s) - expectPresent(t, key, id)(m.Load(key)) - expectDeleted(t, key, id)(m.CompareAndDelete(key, id)) - expectMissing(t, key, 0)(m.Load(key)) - } - for _, s := range testData { - key := makeKey(s) - expectMissing(t, key, 0)(m.Load(key)) - } - }(i) - } - wg.Wait() - }) - t.Run("ConcurrentDeleteSharedKeys", func(t *testing.T) { - m := newMap() - - // Load up the map. - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - } - gmp := runtime.GOMAXPROCS(-1) - var wg sync.WaitGroup - for i := range gmp { - wg.Add(1) - go func(id int) { - defer wg.Done() - - for i, s := range testData { - expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) - m.CompareAndDelete(s, i) - expectMissing(t, s, 0)(m.Load(s)) - } - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - }(i) - } - wg.Wait() - }) -} - -func testAll[K, V comparable](t *testing.T, m *HashTrieMap[K, V], testData map[K]V, yield func(K, V) bool) { - for k, v := range testData { - expectStored(t, k, v)(m.LoadOrStore(k, v)) - } - visited := make(map[K]int) - m.All()(func(key K, got V) bool { - want, ok := testData[key] - if !ok { - t.Errorf("unexpected key %v in map", key) - return false - } - if got != want { - t.Errorf("expected key %v to have value %v, got %v", key, want, got) - return false - } - visited[key]++ - return yield(key, got) - }) - for key, n := range visited { - if n > 1 { - t.Errorf("visited key %v more than once", key) - } - } -} - -func expectPresent[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { - t.Helper() - return func(got V, ok bool) { - t.Helper() - - if !ok { - t.Errorf("expected key %v to be present in map", key) - } - if ok && got != want { - t.Errorf("expected key %v to have value %v, got %v", key, want, got) - } - } -} - -func expectMissing[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { - t.Helper() - if want != *new(V) { - // This is awkward, but the want argument is necessary to smooth over type inference. - // Just make sure the want argument always looks the same. - panic("expectMissing must always have a zero value variable") - } - return func(got V, ok bool) { - t.Helper() - - if ok { - t.Errorf("expected key %v to be missing from map, got value %v", key, got) - } - if !ok && got != want { - t.Errorf("expected missing key %v to be paired with the zero value; got %v", key, got) - } - } -} - -func expectLoaded[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { - t.Helper() - return func(got V, loaded bool) { - t.Helper() - - if !loaded { - t.Errorf("expected key %v to have been loaded, not stored", key) - } - if got != want { - t.Errorf("expected key %v to have value %v, got %v", key, want, got) - } - } -} - -func expectStored[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { - t.Helper() - return func(got V, loaded bool) { - t.Helper() - - if loaded { - t.Errorf("expected inserted key %v to have been stored, not loaded", key) - } - if got != want { - t.Errorf("expected inserted key %v to have value %v, got %v", key, want, got) - } - } -} - -func expectDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { - t.Helper() - return func(deleted bool) { - t.Helper() - - if !deleted { - t.Errorf("expected key %v with value %v to be in map and deleted", key, old) - } - } -} - -func expectNotDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { - t.Helper() - return func(deleted bool) { - t.Helper() - - if deleted { - t.Errorf("expected key %v with value %v to not be in map and thus not deleted", key, old) - } - } -} - -func testDataMap(data []string) map[string]int { - m := make(map[string]int) - for i, s := range data { - m[s] = i - } - return m -} - -var ( - testDataSmall [8]string - testData [128]string - testDataLarge [128 << 10]string -) - -func init() { - for i := range testDataSmall { - testDataSmall[i] = fmt.Sprintf("%b", i) - } - for i := range testData { - testData[i] = fmt.Sprintf("%b", i) - } - for i := range testDataLarge { - testDataLarge[i] = fmt.Sprintf("%b", i) - } -} - -func dumpMap[K, V comparable](ht *HashTrieMap[K, V]) { - dumpNode(ht, &ht.root.node, 0) -} - -func dumpNode[K, V comparable](ht *HashTrieMap[K, V], n *node[K, V], depth int) { - var sb strings.Builder - for range depth { - fmt.Fprintf(&sb, "\t") - } - prefix := sb.String() - if n.isEntry { - e := n.entry() - for e != nil { - fmt.Printf("%s%p [Entry Key=%v Value=%v Overflow=%p, Hash=%016x]\n", prefix, e, e.key, e.value, e.overflow.Load(), ht.keyHash(unsafe.Pointer(&e.key), ht.seed)) - e = e.overflow.Load() - } - return - } - i := n.indirect() - fmt.Printf("%s%p [Indirect Parent=%p Dead=%t Children=[", prefix, i, i.parent, i.dead.Load()) - for j := range i.children { - c := i.children[j].Load() - fmt.Printf("%p", c) - if j != len(i.children)-1 { - fmt.Printf(", ") - } - } - fmt.Printf("]]\n") - for j := range i.children { - c := i.children[j].Load() - if c != nil { - dumpNode(ht, c, depth+1) - } - } -} diff --git a/src/internal/copyright/copyright_test.go b/src/internal/copyright/copyright_test.go new file mode 100644 index 00000000..fa33a944 --- /dev/null +++ b/src/internal/copyright/copyright_test.go @@ -0,0 +1,70 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package copyright + +import ( + "bytes" + "internal/testenv" + "io" + "io/fs" + "os" + "path/filepath" + "testing" +) + +var copyright = []byte("Copyright") + +var permitted = [][]byte{ + []byte("// Code generated by "), + []byte("// Code generated from "), + []byte("// Created by cgo -cdefs"), + []byte("// DO NOT EDIT\n// generated by:"), + []byte("// Empty assembly file"), + []byte("// Generated using cgo"), + []byte("// Original source:\n//\thttp://www.zorinaq.com/papers/md5-amd64.html"), // public domain crypto/md5 + []byte("// created by cgo -cdefs"), + []byte("// go run mkasm.go"), + []byte("// mkerrors"), + []byte("// mksys"), + []byte("// run\n// Code generated by"), // cmd/compile/internal/test/constFold_test.go +} + +func TestCopyright(t *testing.T) { + buf := make([]byte, 2048) + filepath.WalkDir(filepath.Join(testenv.GOROOT(t), "src"), func(path string, d fs.DirEntry, err error) error { + if d.IsDir() && (d.Name() == "testdata" || d.Name() == "vendor") { + return filepath.SkipDir + } + switch filepath.Ext(d.Name()) { + default: + return nil + case ".s", ".go": + // check + } + + f, err := os.Open(path) + if err != nil { + t.Error(err) + return nil + } + defer f.Close() + n, err := f.Read(buf) + if err != nil && err != io.EOF { + t.Error(err) + return nil + } + b := buf[:n] + if bytes.Contains(b, copyright) { + return nil + } + for _, ok := range permitted { + if bytes.HasPrefix(b, ok) { + return nil + } + } + t.Errorf("%s: missing copyright notice", path) + return nil + }) +} diff --git a/src/internal/coverage/cfile/emit.go b/src/internal/coverage/cfile/emit.go index 3993e9cb..47de7778 100644 --- a/src/internal/coverage/cfile/emit.go +++ b/src/internal/coverage/cfile/emit.go @@ -9,8 +9,8 @@ package cfile import ( - "crypto/md5" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/encodecounter" "internal/coverage/encodemeta" @@ -206,7 +206,7 @@ func prepareForMetaEmit() ([]rtcov.CovMetaBlob, error) { } } - h := md5.New() + h := fnv.New128a() tlen := uint64(unsafe.Sizeof(coverage.MetaFileHeader{})) for _, entry := range ml { if _, err := h.Write(entry.Hash[:]); err != nil { diff --git a/src/internal/coverage/cfile/testsupport.go b/src/internal/coverage/cfile/testsupport.go index 3594b32a..adab47fd 100644 --- a/src/internal/coverage/cfile/testsupport.go +++ b/src/internal/coverage/cfile/testsupport.go @@ -109,7 +109,7 @@ func ProcessCoverTestDir(dir string, cfile string, cm string, cpkg string, w io. // Emit text output. if tf != nil { - if err := ts.cf.EmitTextual(tf); err != nil { + if err := ts.cf.EmitTextual(selpkgs, tf); err != nil { return err } tfClosed = true diff --git a/src/internal/coverage/cformat/fmt_test.go b/src/internal/coverage/cformat/fmt_test.go index d296939d..a26de964 100644 --- a/src/internal/coverage/cformat/fmt_test.go +++ b/src/internal/coverage/cformat/fmt_test.go @@ -47,8 +47,8 @@ func TestBasics(t *testing.T) { fm.AddUnit("lit.go", "f3", true, u, 0) } - var b1, b2, b3, b4 strings.Builder - if err := fm.EmitTextual(&b1); err != nil { + var b1, b2, b3, b4, b5 strings.Builder + if err := fm.EmitTextual(nil, &b1); err != nil { t.Fatalf("EmitTextual returned %v", err) } wantText := strings.TrimSpace(` @@ -64,6 +64,18 @@ lit.go:99.0,100.0 1 0`) t.Errorf("emit text: got:\n%s\nwant:\n%s\n", gotText, wantText) } + selected := []string{"my/pack2"} + if err := fm.EmitTextual(selected, &b5); err != nil { + t.Fatalf("EmitTextual returned %v", err) + } + wantText = strings.TrimSpace(` +mode: atomic +lit.go:99.0,100.0 1 0`) + gotText = strings.TrimSpace(b5.String()) + if wantText != gotText { + t.Errorf("emit text: got:\n%s\nwant:\n%s\n", gotText, wantText) + } + // Percent output with no aggregation. noCoverPkg := "" if err := fm.EmitPercent(&b2, nil, noCoverPkg, false, false); err != nil { diff --git a/src/internal/coverage/cformat/format.go b/src/internal/coverage/cformat/format.go index 0f9e981d..01d3109e 100644 --- a/src/internal/coverage/cformat/format.go +++ b/src/internal/coverage/cformat/format.go @@ -24,7 +24,7 @@ package cformat // } // } // myformatter.EmitPercent(os.Stdout, nil, "", true, true) -// myformatter.EmitTextual(somefile) +// myformatter.EmitTextual(nil, somefile) // // These apis are linked into tests that are built with "-cover", and // called at the end of test execution to produce text output or @@ -36,7 +36,9 @@ import ( "internal/coverage" "internal/coverage/cmerge" "io" + "maps" "slices" + "sort" "strings" "text/tabwriter" ) @@ -162,25 +164,31 @@ func (p *pstate) sortUnits(units []extcu) { }) } -// EmitTextual writes the accumulated coverage data in the legacy -// cmd/cover text format to the writer 'w'. We sort the data items by +// EmitTextual writes the accumulated coverage data for 'pkgs' in the legacy +// cmd/cover text format to the writer 'w'; if pkgs is empty, text output +// is emitted for all packages recorded. We sort the data items by // importpath, source file, and line number before emitting (this sorting // is not explicitly mandated by the format, but seems like a good idea // for repeatable/deterministic dumps). -func (fm *Formatter) EmitTextual(w io.Writer) error { +func (fm *Formatter) EmitTextual(pkgs []string, w io.Writer) error { if fm.cm == coverage.CtrModeInvalid { panic("internal error, counter mode unset") } + if len(pkgs) == 0 { + pkgs = make([]string, 0, len(fm.pm)) + for importpath := range fm.pm { + pkgs = append(pkgs, importpath) + } + } if _, err := fmt.Fprintf(w, "mode: %s\n", fm.cm.String()); err != nil { return err } - pkgs := make([]string, 0, len(fm.pm)) - for importpath := range fm.pm { - pkgs = append(pkgs, importpath) - } - slices.Sort(pkgs) + sort.Strings(pkgs) for _, importpath := range pkgs { p := fm.pm[importpath] + if p == nil { + continue + } units := make([]extcu, 0, len(p.unitTable)) for u := range p.unitTable { units = append(units, u) @@ -281,14 +289,8 @@ func (fm *Formatter) EmitFuncs(w io.Writer) error { allStmts := uint64(0) covStmts := uint64(0) - pkgs := make([]string, 0, len(fm.pm)) - for importpath := range fm.pm { - pkgs = append(pkgs, importpath) - } - slices.Sort(pkgs) - // Emit functions for each package, sorted by import path. - for _, importpath := range pkgs { + for _, importpath := range slices.Sorted(maps.Keys(fm.pm)) { p := fm.pm[importpath] if len(p.unitTable) == 0 { continue diff --git a/src/internal/coverage/decodemeta/decodefile.go b/src/internal/coverage/decodemeta/decodefile.go index 96e07659..6f4dd1a3 100644 --- a/src/internal/coverage/decodemeta/decodefile.go +++ b/src/internal/coverage/decodemeta/decodefile.go @@ -12,9 +12,9 @@ package decodemeta import ( "bufio" - "crypto/md5" "encoding/binary" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/slicereader" "internal/coverage/stringtab" @@ -171,8 +171,10 @@ func (r *CoverageMetaFileReader) FileHash() [16]byte { func (r *CoverageMetaFileReader) GetPackageDecoder(pkIdx uint32, payloadbuf []byte) (*CoverageMetaDataDecoder, []byte, error) { pp, err := r.GetPackagePayload(pkIdx, payloadbuf) if r.debug { + h := fnv.New128a() + h.Write(pp) fmt.Fprintf(os.Stderr, "=-= pkidx=%d payload length is %d hash=%s\n", - pkIdx, len(pp), fmt.Sprintf("%x", md5.Sum(pp))) + pkIdx, len(pp), fmt.Sprintf("%x", h.Sum(nil))) } if err != nil { return nil, nil, err diff --git a/src/internal/coverage/encodecounter/encode.go b/src/internal/coverage/encodecounter/encode.go index d5082701..82fd328f 100644 --- a/src/internal/coverage/encodecounter/encode.go +++ b/src/internal/coverage/encodecounter/encode.go @@ -13,6 +13,7 @@ import ( "internal/coverage/stringtab" "internal/coverage/uleb128" "io" + "maps" "os" "slices" ) @@ -122,11 +123,7 @@ func (cfw *CoverageDataWriter) writeSegmentPreamble(args map[string]string, ws * } cfw.csh.StrTabLen = uint32(len(ws.BytesWritten())) - hdrsz - akeys := make([]string, 0, len(args)) - for k := range args { - akeys = append(akeys, k) - } - slices.Sort(akeys) + akeys := slices.Sorted(maps.Keys(args)) wrULEB128 := func(v uint) error { cfw.tmp = cfw.tmp[:0] diff --git a/src/internal/coverage/encodemeta/encode.go b/src/internal/coverage/encodemeta/encode.go index 549b3f55..e8f70fe5 100644 --- a/src/internal/coverage/encodemeta/encode.go +++ b/src/internal/coverage/encodemeta/encode.go @@ -10,10 +10,10 @@ package encodemeta import ( "bytes" - "crypto/md5" "encoding/binary" "fmt" "hash" + "hash/fnv" "internal/coverage" "internal/coverage/stringtab" "internal/coverage/uleb128" @@ -39,7 +39,7 @@ func NewCoverageMetaDataBuilder(pkgpath string, pkgname string, modulepath strin } x := &CoverageMetaDataBuilder{ tmp: make([]byte, 0, 256), - h: md5.New(), + h: fnv.New128a(), } x.stab.InitWriter() x.stab.Lookup("") @@ -188,7 +188,7 @@ func (b *CoverageMetaDataBuilder) Emit(w io.WriteSeeker) ([16]byte, error) { // HashFuncDesc computes an md5 sum of a coverage.FuncDesc and returns // a digest for it. func HashFuncDesc(f *coverage.FuncDesc) [16]byte { - h := md5.New() + h := fnv.New128a() tmp := make([]byte, 0, 32) hashFuncDesc(h, f, tmp) var r [16]byte diff --git a/src/internal/coverage/encodemeta/encodefile.go b/src/internal/coverage/encodemeta/encodefile.go index 38ae46e4..ae7c23ad 100644 --- a/src/internal/coverage/encodemeta/encodefile.go +++ b/src/internal/coverage/encodemeta/encodefile.go @@ -6,9 +6,9 @@ package encodemeta import ( "bufio" - "crypto/md5" "encoding/binary" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/stringtab" "io" @@ -112,7 +112,9 @@ func (m *CoverageMetaFileWriter) Write(finalHash [16]byte, blobs [][]byte, mode // Now emit blobs themselves. for k, blob := range blobs { if m.debug { - fmt.Fprintf(os.Stderr, "=+= writing blob %d len %d at off=%d hash %s\n", k, len(blob), off2, fmt.Sprintf("%x", md5.Sum(blob))) + h := fnv.New128a() + h.Write(blob) + fmt.Fprintf(os.Stderr, "=+= writing blob %d len %d at off=%d hash %s\n", k, len(blob), off2, fmt.Sprintf("%x", h.Sum(nil))) } if _, err = m.w.Write(blob); err != nil { return fmt.Errorf("error writing %s: %v", m.mfname, err) diff --git a/src/internal/coverage/pkid.go b/src/internal/coverage/pkid.go index 86ff3025..f68523a3 100644 --- a/src/internal/coverage/pkid.go +++ b/src/internal/coverage/pkid.go @@ -26,9 +26,9 @@ package coverage // slot: 1 path='internal/goarch' hard-coded id: 2 // slot: 2 path='internal/runtime/atomic' hard-coded id: 3 // slot: 3 path='internal/goos' -// slot: 4 path='runtime/internal/sys' hard-coded id: 5 +// slot: 4 path='internal/runtime/sys' hard-coded id: 5 // slot: 5 path='internal/abi' hard-coded id: 4 -// slot: 6 path='runtime/internal/math' hard-coded id: 6 +// slot: 6 path='internal/runtime/math' hard-coded id: 6 // slot: 7 path='internal/bytealg' hard-coded id: 7 // slot: 8 path='internal/goexperiment' // slot: 9 path='internal/runtime/syscall' hard-coded id: 8 @@ -50,12 +50,14 @@ var rtPkgs = [...]string{ "internal/runtime/atomic", "internal/goos", "internal/chacha8rand", - "runtime/internal/sys", + "internal/runtime/sys", "internal/abi", - "runtime/internal/math", + "internal/runtime/maps", + "internal/runtime/math", "internal/bytealg", "internal/goexperiment", "internal/runtime/syscall", + "internal/stringslite", "runtime", } diff --git a/src/internal/coverage/pods/pods_test.go b/src/internal/coverage/pods/pods_test.go index 69c16e00..eed01698 100644 --- a/src/internal/coverage/pods/pods_test.go +++ b/src/internal/coverage/pods/pods_test.go @@ -5,8 +5,8 @@ package pods_test import ( - "crypto/md5" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/pods" "os" @@ -35,13 +35,17 @@ func TestPodCollection(t *testing.T) { } mkmeta := func(dir string, tag string) string { - hash := md5.Sum([]byte(tag)) + h := fnv.New128a() + h.Write([]byte(tag)) + hash := h.Sum(nil) fn := fmt.Sprintf("%s.%x", coverage.MetaFilePref, hash) return mkfile(dir, fn) } mkcounter := func(dir string, tag string, nt int, pid int) string { - hash := md5.Sum([]byte(tag)) + h := fnv.New128a() + h.Write([]byte(tag)) + hash := h.Sum(nil) fn := fmt.Sprintf(coverage.CounterFileTempl, coverage.CounterFilePref, hash, pid, nt) return mkfile(dir, fn) } @@ -112,16 +116,16 @@ func TestPodCollection(t *testing.T) { } expected := []string{ - `o1/covmeta.ae7be26cdaa742ca148068d5ac90eaca [ -o1/covcounters.ae7be26cdaa742ca148068d5ac90eaca.40.2 o:0 -o1/covcounters.ae7be26cdaa742ca148068d5ac90eaca.41.2 o:0 -o1/covcounters.ae7be26cdaa742ca148068d5ac90eaca.42.1 o:0 -o2/covcounters.ae7be26cdaa742ca148068d5ac90eaca.35.11 o:1 + `o1/covmeta.0880952782ab1be95aa0733055a4d06b [ +o1/covcounters.0880952782ab1be95aa0733055a4d06b.40.2 o:0 +o1/covcounters.0880952782ab1be95aa0733055a4d06b.41.2 o:0 +o1/covcounters.0880952782ab1be95aa0733055a4d06b.42.1 o:0 +o2/covcounters.0880952782ab1be95aa0733055a4d06b.35.11 o:1 ]`, - `o2/covmeta.aaf2f89992379705dac844c0a2a1d45f [ -o2/covcounters.aaf2f89992379705dac844c0a2a1d45f.36.3 o:1 -o2/covcounters.aaf2f89992379705dac844c0a2a1d45f.37.2 o:1 -o2/covcounters.aaf2f89992379705dac844c0a2a1d45f.38.1 o:1 + `o2/covmeta.0880952783ab1be95aa0733055a4d1a6 [ +o2/covcounters.0880952783ab1be95aa0733055a4d1a6.36.3 o:1 +o2/covcounters.0880952783ab1be95aa0733055a4d1a6.37.2 o:1 +o2/covcounters.0880952783ab1be95aa0733055a4d1a6.38.1 o:1 ]`, } for k, exp := range expected { diff --git a/src/internal/cpu/cpu.go b/src/internal/cpu/cpu.go index 9be280c6..cd3db105 100644 --- a/src/internal/cpu/cpu.go +++ b/src/internal/cpu/cpu.go @@ -37,6 +37,7 @@ var X86 struct { HasBMI1 bool HasBMI2 bool HasERMS bool + HasFSRM bool HasFMA bool HasOSXSAVE bool HasPCLMULQDQ bool @@ -72,10 +73,22 @@ var ARM64 struct { HasCRC32 bool HasATOMICS bool HasCPUID bool + HasDIT bool IsNeoverse bool _ CacheLinePad } +// The booleans in Loong64 contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +var Loong64 struct { + _ CacheLinePad + HasLSX bool // support 128-bit vector extension + HasCRC32 bool // support CRC instruction + HasLAMCAS bool // support AMCAS[_DB].{B/H/W/D} + HasLAM_BH bool // support AM{SWAP/ADD}[_DB].{B/H} instruction + _ CacheLinePad +} + var MIPS64X struct { _ CacheLinePad HasMSA bool // MIPS SIMD architecture @@ -127,6 +140,7 @@ var S390X struct { //go:linkname X86 //go:linkname ARM //go:linkname ARM64 +//go:linkname Loong64 //go:linkname MIPS64X //go:linkname PPC64 //go:linkname S390X diff --git a/src/internal/cpu/cpu_arm64.go b/src/internal/cpu/cpu_arm64.go index 4a302f27..1365991e 100644 --- a/src/internal/cpu/cpu_arm64.go +++ b/src/internal/cpu/cpu_arm64.go @@ -28,13 +28,15 @@ func doinit() { func getisar0() uint64 +func getpfr0() uint64 + func getMIDR() uint64 func extractBits(data uint64, start, end uint) uint { return (uint)(data>>start) & ((1 << (end - start + 1)) - 1) } -func parseARM64SystemRegisters(isar0 uint64) { +func parseARM64SystemRegisters(isar0, pfr0 uint64) { // ID_AA64ISAR0_EL1 switch extractBits(isar0, 4, 7) { case 1: @@ -66,4 +68,9 @@ func parseARM64SystemRegisters(isar0 uint64) { case 2: ARM64.HasATOMICS = true } + + switch extractBits(pfr0, 48, 51) { + case 1: + ARM64.HasDIT = true + } } diff --git a/src/internal/cpu/cpu_arm64.s b/src/internal/cpu/cpu_arm64.s index d6e7f443..96075610 100644 --- a/src/internal/cpu/cpu_arm64.s +++ b/src/internal/cpu/cpu_arm64.s @@ -11,6 +11,13 @@ TEXT ·getisar0(SB),NOSPLIT,$0 MOVD R0, ret+0(FP) RET +// func getpfr0() uint64 +TEXT ·getpfr0(SB),NOSPLIT,$0-8 + // get Processor Feature Register 0 into R0 + MRS ID_AA64PFR0_EL1, R0 + MOVD R0, ret+0(FP) + RET + // func getMIDR() uint64 TEXT ·getMIDR(SB), NOSPLIT, $0-8 MRS MIDR_EL1, R0 diff --git a/src/internal/cpu/cpu_arm64_darwin.go b/src/internal/cpu/cpu_arm64_darwin.go index 2507780e..57cf631e 100644 --- a/src/internal/cpu/cpu_arm64_darwin.go +++ b/src/internal/cpu/cpu_arm64_darwin.go @@ -12,6 +12,7 @@ func osInit() { ARM64.HasATOMICS = sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00")) ARM64.HasCRC32 = sysctlEnabled([]byte("hw.optional.armv8_crc32\x00")) ARM64.HasSHA512 = sysctlEnabled([]byte("hw.optional.armv8_2_sha512\x00")) + ARM64.HasDIT = sysctlEnabled([]byte("hw.optional.arm.FEAT_DIT\x00")) // There are no hw.optional sysctl values for the below features on Mac OS 11.0 // to detect their supported state dynamically. Assume the CPU features that diff --git a/src/internal/cpu/cpu_arm64_freebsd.go b/src/internal/cpu/cpu_arm64_freebsd.go index 96ed359c..c339e6f2 100644 --- a/src/internal/cpu/cpu_arm64_freebsd.go +++ b/src/internal/cpu/cpu_arm64_freebsd.go @@ -9,6 +9,7 @@ package cpu func osInit() { // Retrieve info from system register ID_AA64ISAR0_EL1. isar0 := getisar0() + prf0 := getpfr0() - parseARM64SystemRegisters(isar0) + parseARM64SystemRegisters(isar0, prf0) } diff --git a/src/internal/cpu/cpu_arm64_hwcap.go b/src/internal/cpu/cpu_arm64_hwcap.go index 34edf3ee..cdc1d89c 100644 --- a/src/internal/cpu/cpu_arm64_hwcap.go +++ b/src/internal/cpu/cpu_arm64_hwcap.go @@ -31,6 +31,7 @@ const ( hwcap_ATOMICS = 1 << 8 hwcap_CPUID = 1 << 11 hwcap_SHA512 = 1 << 21 + hwcap_DIT = 1 << 24 ) func hwcapInit(os string) { @@ -44,6 +45,7 @@ func hwcapInit(os string) { ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32) ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID) ARM64.HasSHA512 = isSet(HWCap, hwcap_SHA512) + ARM64.HasDIT = isSet(HWCap, hwcap_DIT) // The Samsung S9+ kernel reports support for atomics, but not all cores // actually support them, resulting in SIGILL. See issue #28431. diff --git a/src/internal/cpu/cpu_arm64_openbsd.go b/src/internal/cpu/cpu_arm64_openbsd.go index 12593098..6cc69c95 100644 --- a/src/internal/cpu/cpu_arm64_openbsd.go +++ b/src/internal/cpu/cpu_arm64_openbsd.go @@ -13,6 +13,7 @@ const ( // From OpenBSD's machine/cpu.h. _CPU_ID_AA64ISAR0 = 2 _CPU_ID_AA64ISAR1 = 3 + _CPU_ID_AA64PFR0 = 8 ) //go:noescape @@ -24,5 +25,11 @@ func osInit() { if !ok { return } - parseARM64SystemRegisters(isar0) + // Get ID_AA64PFR0 from sysctl. + pfr0, ok := sysctlUint64([]uint32{_CTL_MACHDEP, _CPU_ID_AA64PFR0}) + if !ok { + return + } + + parseARM64SystemRegisters(isar0, pfr0) } diff --git a/src/internal/cpu/cpu_loong64.go b/src/internal/cpu/cpu_loong64.go index 1c90c24f..92583d0b 100644 --- a/src/internal/cpu/cpu_loong64.go +++ b/src/internal/cpu/cpu_loong64.go @@ -10,4 +10,45 @@ package cpu // We choose 64 because Loongson 3A5000 the L1 Dcache is 4-way 256-line 64-byte-per-line. const CacheLinePadSize = 64 -func doinit() {} +// Bit fields for CPUCFG registers, Related reference documents: +// https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_cpucfg +const ( + // CPUCFG1 bits + cpucfg1_CRC32 = 1 << 25 + + // CPUCFG2 bits + cpucfg2_LAM_BH = 1 << 27 + cpucfg2_LAMCAS = 1 << 28 +) + +// get_cpucfg is implemented in cpu_loong64.s. +func get_cpucfg(reg uint32) uint32 + +func doinit() { + options = []option{ + {Name: "lsx", Feature: &Loong64.HasLSX}, + {Name: "crc32", Feature: &Loong64.HasCRC32}, + {Name: "lamcas", Feature: &Loong64.HasLAMCAS}, + {Name: "lam_bh", Feature: &Loong64.HasLAM_BH}, + } + + // The CPUCFG data on Loong64 only reflects the hardware capabilities, + // not the kernel support status, so features such as LSX and LASX that + // require kernel support cannot be obtained from the CPUCFG data. + // + // These features only require hardware capability support and do not + // require kernel specific support, so they can be obtained directly + // through CPUCFG + cfg1 := get_cpucfg(1) + cfg2 := get_cpucfg(2) + + Loong64.HasCRC32 = cfgIsSet(cfg1, cpucfg1_CRC32) + Loong64.HasLAMCAS = cfgIsSet(cfg2, cpucfg2_LAM_BH) + Loong64.HasLAM_BH = cfgIsSet(cfg2, cpucfg2_LAMCAS) + + osInit() +} + +func cfgIsSet(cfg uint32, val uint32) bool { + return cfg&val != 0 +} diff --git a/src/internal/cpu/cpu_loong64.s b/src/internal/cpu/cpu_loong64.s new file mode 100644 index 00000000..f02a2780 --- /dev/null +++ b/src/internal/cpu/cpu_loong64.s @@ -0,0 +1,12 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func get_cpucfg(reg uint32) uint32 +TEXT ·get_cpucfg(SB), NOSPLIT|NOFRAME, $0-12 + MOVW reg+0(FP), R5 + CPUCFG R5, R4 + MOVW R4, ret+8(FP) + RET diff --git a/src/internal/cpu/cpu_loong64_hwcap.go b/src/internal/cpu/cpu_loong64_hwcap.go new file mode 100644 index 00000000..58397ada --- /dev/null +++ b/src/internal/cpu/cpu_loong64_hwcap.go @@ -0,0 +1,26 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && linux + +package cpu + +// This is initialized by archauxv and should not be changed after it is +// initialized. +var HWCap uint + +// HWCAP bits. These are exposed by the Linux kernel. +const ( + hwcap_LOONGARCH_LSX = 1 << 4 +) + +func hwcapInit() { + // TODO: Features that require kernel support like LSX and LASX can + // be detected here once needed in std library or by the compiler. + Loong64.HasLSX = hwcIsSet(HWCap, hwcap_LOONGARCH_LSX) +} + +func hwcIsSet(hwc uint, val uint) bool { + return hwc&val != 0 +} diff --git a/src/internal/cpu/cpu_loong64_linux.go b/src/internal/cpu/cpu_loong64_linux.go new file mode 100644 index 00000000..73bc384a --- /dev/null +++ b/src/internal/cpu/cpu_loong64_linux.go @@ -0,0 +1,11 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && linux + +package cpu + +func osInit() { + hwcapInit() +} diff --git a/src/internal/cpu/cpu_x86.go b/src/internal/cpu/cpu_x86.go index 2b629d4d..ee812076 100644 --- a/src/internal/cpu/cpu_x86.go +++ b/src/internal/cpu/cpu_x86.go @@ -40,7 +40,8 @@ const ( cpuid_SHA = 1 << 29 cpuid_AVX512BW = 1 << 30 cpuid_AVX512VL = 1 << 31 - + // edx bits + cpuid_FSRM = 1 << 4 // edx bits for CPUID 0x80000001 cpuid_RDTSCP = 1 << 27 ) @@ -52,6 +53,7 @@ func doinit() { {Name: "adx", Feature: &X86.HasADX}, {Name: "aes", Feature: &X86.HasAES}, {Name: "erms", Feature: &X86.HasERMS}, + {Name: "fsrm", Feature: &X86.HasFSRM}, {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ}, {Name: "rdtscp", Feature: &X86.HasRDTSCP}, {Name: "sha", Feature: &X86.HasSHA}, @@ -137,7 +139,7 @@ func doinit() { return } - _, ebx7, _, _ := cpuid(7, 0) + _, ebx7, _, edx7 := cpuid(7, 0) X86.HasBMI1 = isSet(ebx7, cpuid_BMI1) X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX X86.HasBMI2 = isSet(ebx7, cpuid_BMI2) @@ -151,6 +153,8 @@ func doinit() { X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL) } + X86.HasFSRM = isSet(edx7, cpuid_FSRM) + var maxExtendedInformation uint32 maxExtendedInformation, _, _, _ = cpuid(0x80000000, 0) diff --git a/src/internal/dag/alg_test.go b/src/internal/dag/alg_test.go index e5ea8b6a..0659eb18 100644 --- a/src/internal/dag/alg_test.go +++ b/src/internal/dag/alg_test.go @@ -5,7 +5,7 @@ package dag import ( - "reflect" + "slices" "strings" "testing" ) @@ -26,7 +26,7 @@ func TestTopo(t *testing.T) { // // "a" is a leaf. wantNodes := strings.Fields("d c b a") - if !reflect.DeepEqual(wantNodes, got) { + if !slices.Equal(wantNodes, got) { t.Fatalf("want topo sort %v, got %v", wantNodes, got) } } diff --git a/src/internal/dag/parse_test.go b/src/internal/dag/parse_test.go index b2520c36..dda304ad 100644 --- a/src/internal/dag/parse_test.go +++ b/src/internal/dag/parse_test.go @@ -5,7 +5,7 @@ package dag import ( - "reflect" + "slices" "strings" "testing" ) @@ -52,7 +52,7 @@ func TestParse(t *testing.T) { g := mustParse(t, diamond) wantNodes := strings.Fields("a b c d") - if !reflect.DeepEqual(wantNodes, g.Nodes) { + if !slices.Equal(wantNodes, g.Nodes) { t.Fatalf("want nodes %v, got %v", wantNodes, g.Nodes) } diff --git a/src/internal/exportdata/exportdata.go b/src/internal/exportdata/exportdata.go new file mode 100644 index 00000000..861a47f4 --- /dev/null +++ b/src/internal/exportdata/exportdata.go @@ -0,0 +1,355 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package exportdata implements common utilities for finding +// and reading gc-generated object files. +package exportdata + +// This file should be kept in sync with src/cmd/compile/internal/gc/obj.go . + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "go/build" + "internal/saferio" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" +) + +// ReadUnified reads the contents of the unified export data from a reader r +// that contains the contents of a GC-created archive file. +// +// On success, the reader will be positioned after the end-of-section marker "\n$$\n". +// +// Supported GC-created archive files have 4 layers of nesting: +// - An archive file containing a package definition file. +// - The package definition file contains headers followed by a data section. +// Headers are lines (≤ 4kb) that do not start with "$$". +// - The data section starts with "$$B\n" followed by export data followed +// by an end of section marker "\n$$\n". (The section start "$$\n" is no +// longer supported.) +// - The export data starts with a format byte ('u') followed by the in +// the given format. (See ReadExportDataHeader for older formats.) +// +// Putting this together, the bytes in a GC-created archive files are expected +// to look like the following. +// See cmd/internal/archive for more details on ar file headers. +// +// | \n | ar file signature +// | __.PKGDEF...size...\n | ar header for __.PKGDEF including size. +// | go object <...>\n | objabi header +// | \n | other headers such as build id +// | $$B\n | binary format marker +// | u\n | unified export +// | $$\n | end-of-section marker +// | [optional padding] | padding byte (0x0A) if size is odd +// | [ar file header] | other ar files +// | [ar file data] | +func ReadUnified(r *bufio.Reader) (data []byte, err error) { + // We historically guaranteed headers at the default buffer size (4096) work. + // This ensures we can use ReadSlice throughout. + const minBufferSize = 4096 + r = bufio.NewReaderSize(r, minBufferSize) + + size, err := FindPackageDefinition(r) + if err != nil { + return + } + n := size + + objapi, headers, err := ReadObjectHeaders(r) + if err != nil { + return + } + n -= len(objapi) + for _, h := range headers { + n -= len(h) + } + + hdrlen, err := ReadExportDataHeader(r) + if err != nil { + return + } + n -= hdrlen + + // size also includes the end of section marker. Remove that many bytes from the end. + const marker = "\n$$\n" + n -= len(marker) + + if n < 0 { + err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", size, n) + return + } + + // Read n bytes from buf. + data, err = saferio.ReadData(r, uint64(n)) + if err != nil { + return + } + + // Check for marker at the end. + var suffix [len(marker)]byte + _, err = io.ReadFull(r, suffix[:]) + if err != nil { + return + } + if s := string(suffix[:]); s != marker { + err = fmt.Errorf("read %q instead of end-of-section marker (%q)", s, marker) + return + } + + return +} + +// FindPackageDefinition positions the reader r at the beginning of a package +// definition file ("__.PKGDEF") within a GC-created archive by reading +// from it, and returns the size of the package definition file in the archive. +// +// The reader must be positioned at the start of the archive file before calling +// this function, and "__.PKGDEF" is assumed to be the first file in the archive. +// +// See cmd/internal/archive for details on the archive format. +func FindPackageDefinition(r *bufio.Reader) (size int, err error) { + // Uses ReadSlice to limit risk of malformed inputs. + + // Read first line to make sure this is an object file. + line, err := r.ReadSlice('\n') + if err != nil { + err = fmt.Errorf("can't find export data (%v)", err) + return + } + + // Is the first line an archive file signature? + if string(line) != "!\n" { + err = fmt.Errorf("not the start of an archive file (%q)", line) + return + } + + // package export block should be first + size = readArchiveHeader(r, "__.PKGDEF") + if size <= 0 { + err = fmt.Errorf("not a package file") + return + } + + return +} + +// ReadObjectHeaders reads object headers from the reader. Object headers are +// lines that do not start with an end-of-section marker "$$". The first header +// is the objabi header. On success, the reader will be positioned at the beginning +// of the end-of-section marker. +// +// It returns an error if any header does not fit in r.Size() bytes. +func ReadObjectHeaders(r *bufio.Reader) (objapi string, headers []string, err error) { + // line is a temporary buffer for headers. + // Use bounded reads (ReadSlice, Peek) to limit risk of malformed inputs. + var line []byte + + // objapi header should be the first line + if line, err = r.ReadSlice('\n'); err != nil { + err = fmt.Errorf("can't find export data (%v)", err) + return + } + objapi = string(line) + + // objapi header begins with "go object ". + if !strings.HasPrefix(objapi, "go object ") { + err = fmt.Errorf("not a go object file: %s", objapi) + return + } + + // process remaining object header lines + for { + // check for an end of section marker "$$" + line, err = r.Peek(2) + if err != nil { + return + } + if string(line) == "$$" { + return // stop + } + + // read next header + line, err = r.ReadSlice('\n') + if err != nil { + return + } + headers = append(headers, string(line)) + } +} + +// ReadExportDataHeader reads the export data header and format from r. +// It returns the number of bytes read, or an error if the format is no longer +// supported or it failed to read. +// +// The only currently supported format is binary export data in the +// unified export format. +func ReadExportDataHeader(r *bufio.Reader) (n int, err error) { + // Read export data header. + line, err := r.ReadSlice('\n') + if err != nil { + return + } + + hdr := string(line) + switch hdr { + case "$$\n": + err = fmt.Errorf("old textual export format no longer supported (recompile package)") + return + + case "$$B\n": + var format byte + format, err = r.ReadByte() + if err != nil { + return + } + // The unified export format starts with a 'u'. + switch format { + case 'u': + default: + // Older no longer supported export formats include: + // indexed export format which started with an 'i'; and + // the older binary export format which started with a 'c', + // 'd', or 'v' (from "version"). + err = fmt.Errorf("binary export format %q is no longer supported (recompile package)", format) + return + } + + default: + err = fmt.Errorf("unknown export data header: %q", hdr) + return + } + + n = len(hdr) + 1 // + 1 is for 'u' + return +} + +// FindPkg returns the filename and unique package id for an import +// path based on package information provided by build.Import (using +// the build.Default build.Context). A relative srcDir is interpreted +// relative to the current working directory. +func FindPkg(path, srcDir string) (filename, id string, err error) { + if path == "" { + return "", "", errors.New("path is empty") + } + + var noext string + switch { + default: + // "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x" + // Don't require the source files to be present. + if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282 + srcDir = abs + } + var bp *build.Package + bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary) + if bp.PkgObj == "" { + if bp.Goroot && bp.Dir != "" { + filename, err = lookupGorootExport(bp.Dir) + if err == nil { + _, err = os.Stat(filename) + } + if err == nil { + return filename, bp.ImportPath, nil + } + } + goto notfound + } else { + noext = strings.TrimSuffix(bp.PkgObj, ".a") + } + id = bp.ImportPath + + case build.IsLocalImport(path): + // "./x" -> "/this/directory/x.ext", "/this/directory/x" + noext = filepath.Join(srcDir, path) + id = noext + + case filepath.IsAbs(path): + // for completeness only - go/build.Import + // does not support absolute imports + // "/x" -> "/x.ext", "/x" + noext = path + id = path + } + + if false { // for debugging + if path != id { + fmt.Printf("%s -> %s\n", path, id) + } + } + + // try extensions + for _, ext := range pkgExts { + filename = noext + ext + f, statErr := os.Stat(filename) + if statErr == nil && !f.IsDir() { + return filename, id, nil + } + if err == nil { + err = statErr + } + } + +notfound: + if err == nil { + return "", path, fmt.Errorf("can't find import: %q", path) + } + return "", path, fmt.Errorf("can't find import: %q: %w", path, err) +} + +var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension + +var exportMap sync.Map // package dir → func() (string, error) + +// lookupGorootExport returns the location of the export data +// (normally found in the build cache, but located in GOROOT/pkg +// in prior Go releases) for the package located in pkgDir. +// +// (We use the package's directory instead of its import path +// mainly to simplify handling of the packages in src/vendor +// and cmd/vendor.) +func lookupGorootExport(pkgDir string) (string, error) { + f, ok := exportMap.Load(pkgDir) + if !ok { + var ( + listOnce sync.Once + exportPath string + err error + ) + f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) { + listOnce.Do(func() { + cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir) + cmd.Dir = build.Default.GOROOT + cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT) + var output []byte + output, err = cmd.Output() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { + err = errors.New(string(ee.Stderr)) + } + return + } + + exports := strings.Split(string(bytes.TrimSpace(output)), "\n") + if len(exports) != 1 { + err = fmt.Errorf("go list reported %d exports; expected 1", len(exports)) + return + } + + exportPath = exports[0] + }) + + return exportPath, err + }) + } + + return f.(func() (string, error))() +} diff --git a/src/internal/exportdata/support.go b/src/internal/exportdata/support.go new file mode 100644 index 00000000..a06401db --- /dev/null +++ b/src/internal/exportdata/support.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains support functions for exportdata. + +package exportdata + +import ( + "bufio" + "io" + "strconv" + "strings" +) + +// Copy of cmd/internal/archive.ReadHeader. +func readArchiveHeader(b *bufio.Reader, name string) int { + // architecture-independent object file output + const HeaderSize = 60 + + var buf [HeaderSize]byte + if _, err := io.ReadFull(b, buf[:]); err != nil { + return -1 + } + aname := strings.Trim(string(buf[0:16]), " ") + if !strings.HasPrefix(aname, name) { + return -1 + } + asize := strings.Trim(string(buf[48:58]), " ") + i, _ := strconv.Atoi(asize) + return i +} diff --git a/src/internal/fuzz/mutators_byteslice_test.go b/src/internal/fuzz/mutators_byteslice_test.go index 78869678..b12ef6cb 100644 --- a/src/internal/fuzz/mutators_byteslice_test.go +++ b/src/internal/fuzz/mutators_byteslice_test.go @@ -6,6 +6,7 @@ package fuzz import ( "bytes" + "fmt" "testing" ) @@ -33,12 +34,6 @@ func (mr *mockRand) uint32n(n uint32) uint32 { return uint32(c) % n } -func (mr *mockRand) exp2() int { - c := mr.values[mr.counter] - mr.counter++ - return c -} - func (mr *mockRand) bool() bool { b := mr.b mr.b = !mr.b @@ -184,3 +179,43 @@ func TestByteSliceMutators(t *testing.T) { }) } } + +func BenchmarkByteSliceMutators(b *testing.B) { + tests := [...]struct { + name string + mutator func(*mutator, []byte) []byte + }{ + {"RemoveBytes", byteSliceRemoveBytes}, + {"InsertRandomBytes", byteSliceInsertRandomBytes}, + {"DuplicateBytes", byteSliceDuplicateBytes}, + {"OverwriteBytes", byteSliceOverwriteBytes}, + {"BitFlip", byteSliceBitFlip}, + {"XORByte", byteSliceXORByte}, + {"SwapByte", byteSliceSwapByte}, + {"ArithmeticUint8", byteSliceArithmeticUint8}, + {"ArithmeticUint16", byteSliceArithmeticUint16}, + {"ArithmeticUint32", byteSliceArithmeticUint32}, + {"ArithmeticUint64", byteSliceArithmeticUint64}, + {"OverwriteInterestingUint8", byteSliceOverwriteInterestingUint8}, + {"OverwriteInterestingUint16", byteSliceOverwriteInterestingUint16}, + {"OverwriteInterestingUint32", byteSliceOverwriteInterestingUint32}, + {"InsertConstantBytes", byteSliceInsertConstantBytes}, + {"OverwriteConstantBytes", byteSliceOverwriteConstantBytes}, + {"ShuffleBytes", byteSliceShuffleBytes}, + {"SwapBytes", byteSliceSwapBytes}, + } + + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + for size := 64; size <= 1024; size *= 2 { + b.Run(fmt.Sprintf("%d", size), func(b *testing.B) { + m := &mutator{r: newPcgRand()} + input := make([]byte, size) + for i := 0; i < b.N; i++ { + tc.mutator(m, input) + } + }) + } + }) + } +} diff --git a/src/internal/fuzz/pcg.go b/src/internal/fuzz/pcg.go index dc07b9f5..b8251043 100644 --- a/src/internal/fuzz/pcg.go +++ b/src/internal/fuzz/pcg.go @@ -17,7 +17,6 @@ type mutatorRand interface { uint32() uint32 intn(int) int uint32n(uint32) uint32 - exp2() int bool() bool save(randState, randInc *uint64) @@ -123,11 +122,6 @@ func (r *pcgRand) uint32n(n uint32) uint32 { return uint32(prod >> 32) } -// exp2 generates n with probability 1/2^(n+1). -func (r *pcgRand) exp2() int { - return bits.TrailingZeros32(r.uint32()) -} - // bool generates a random bool. func (r *pcgRand) bool() bool { return r.uint32()&1 == 0 diff --git a/src/internal/fuzz/worker.go b/src/internal/fuzz/worker.go index e8a74217..9ee2f272 100644 --- a/src/internal/fuzz/worker.go +++ b/src/internal/fuzz/worker.go @@ -682,7 +682,7 @@ func (ws *workerServer) serve(ctx context.Context) error { } // chainedMutations is how many mutations are applied before the worker -// resets the input to it's original state. +// resets the input to its original state. // NOTE: this number was picked without much thought. It is low enough that // it seems to create a significant diversity in mutated inputs. We may want // to consider looking into this more closely once we have a proper performance diff --git a/src/internal/goarch/gengoarch.go b/src/internal/goarch/gengoarch.go index 0b0be5cd..a52936ef 100644 --- a/src/internal/goarch/gengoarch.go +++ b/src/internal/goarch/gengoarch.go @@ -17,11 +17,11 @@ import ( var goarches []string func main() { - data, err := os.ReadFile("../../go/build/syslist.go") + data, err := os.ReadFile("../../internal/syslist/syslist.go") if err != nil { log.Fatal(err) } - const goarchPrefix = `var knownArch = map[string]bool{` + const goarchPrefix = `var KnownArch = map[string]bool{` inGOARCH := false for _, line := range strings.Split(string(data), "\n") { if strings.HasPrefix(line, goarchPrefix) { diff --git a/src/internal/godebug/godebug_test.go b/src/internal/godebug/godebug_test.go index 5d426fd5..fe1e6722 100644 --- a/src/internal/godebug/godebug_test.go +++ b/src/internal/godebug/godebug_test.go @@ -11,7 +11,6 @@ import ( "internal/testenv" "os" "os/exec" - "reflect" "runtime/metrics" "slices" "strings" @@ -110,6 +109,9 @@ func TestCmdBisect(t *testing.T) { var want []string src, err := os.ReadFile("godebug_test.go") + if err != nil { + t.Fatal(err) + } for i, line := range strings.Split(string(src), "\n") { if strings.Contains(line, "BISECT"+" "+"BUG") { want = append(want, fmt.Sprintf("godebug_test.go:%d", i+1)) @@ -125,7 +127,7 @@ func TestCmdBisect(t *testing.T) { } slices.Sort(have) - if !reflect.DeepEqual(have, want) { + if !slices.Equal(have, want) { t.Errorf("bad bisect output:\nhave %v\nwant %v\ncomplete output:\n%s", have, want, string(out)) } } diff --git a/src/internal/godebugs/table.go b/src/internal/godebugs/table.go index a802ac9c..9c48a923 100644 --- a/src/internal/godebugs/table.go +++ b/src/internal/godebugs/table.go @@ -26,10 +26,12 @@ type Info struct { // (Otherwise the test in this package will fail.) var All = []Info{ {Name: "asynctimerchan", Package: "time", Changed: 23, Old: "1"}, + {Name: "dataindependenttiming", Package: "crypto/subtle", Opaque: true}, {Name: "execerrdot", Package: "os/exec"}, {Name: "gocachehash", Package: "cmd/go"}, {Name: "gocachetest", Package: "cmd/go"}, {Name: "gocacheverify", Package: "cmd/go"}, + {Name: "gotestjsonbuildtext", Package: "cmd/go", Changed: 24, Old: "1"}, {Name: "gotypesalias", Package: "go/types", Changed: 23, Old: "0"}, {Name: "http2client", Package: "net/http"}, {Name: "http2debug", Package: "net/http", Opaque: true}, @@ -42,25 +44,27 @@ var All = []Info{ //{Name: "multipartfiles", Package: "mime/multipart"}, {Name: "multipartmaxheaders", Package: "mime/multipart"}, {Name: "multipartmaxparts", Package: "mime/multipart"}, - {Name: "multipathtcp", Package: "net"}, + {Name: "multipathtcp", Package: "net", Changed: 24, Old: "0"}, {Name: "netdns", Package: "net", Opaque: true}, {Name: "netedns0", Package: "net", Changed: 19, Old: "0"}, {Name: "panicnil", Package: "runtime", Changed: 21, Old: "1"}, {Name: "randautoseed", Package: "math/rand"}, + {Name: "randseednop", Package: "math/rand", Changed: 24, Old: "0"}, + {Name: "rsa1024min", Package: "crypto/rsa", Changed: 24, Old: "0"}, {Name: "tarinsecurepath", Package: "archive/tar"}, {Name: "tls10server", Package: "crypto/tls", Changed: 22, Old: "1"}, {Name: "tls3des", Package: "crypto/tls", Changed: 23, Old: "1"}, - {Name: "tlskyber", Package: "crypto/tls", Changed: 23, Old: "0", Opaque: true}, {Name: "tlsmaxrsasize", Package: "crypto/tls"}, + {Name: "tlsmlkem", Package: "crypto/tls", Changed: 24, Old: "0", Opaque: true}, {Name: "tlsrsakex", Package: "crypto/tls", Changed: 22, Old: "1"}, {Name: "tlsunsafeekm", Package: "crypto/tls", Changed: 22, Old: "1"}, {Name: "winreadlinkvolume", Package: "os", Changed: 22, Old: "0"}, {Name: "winsymlink", Package: "os", Changed: 22, Old: "0"}, {Name: "x509keypairleaf", Package: "crypto/tls", Changed: 23, Old: "0"}, {Name: "x509negativeserial", Package: "crypto/x509", Changed: 23, Old: "1"}, - {Name: "x509sha1", Package: "crypto/x509"}, + {Name: "x509rsacrt", Package: "crypto/x509", Changed: 24, Old: "0"}, {Name: "x509usefallbackroots", Package: "crypto/x509"}, - {Name: "x509usepolicies", Package: "crypto/x509"}, + {Name: "x509usepolicies", Package: "crypto/x509", Changed: 24, Old: "0"}, {Name: "zipinsecurepath", Package: "archive/zip"}, } diff --git a/src/internal/goexperiment/exp_spinbitmutex_off.go b/src/internal/goexperiment/exp_spinbitmutex_off.go new file mode 100644 index 00000000..776b0dc7 --- /dev/null +++ b/src/internal/goexperiment/exp_spinbitmutex_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.spinbitmutex + +package goexperiment + +const SpinbitMutex = false +const SpinbitMutexInt = 0 diff --git a/src/internal/goexperiment/exp_spinbitmutex_on.go b/src/internal/goexperiment/exp_spinbitmutex_on.go new file mode 100644 index 00000000..8468030c --- /dev/null +++ b/src/internal/goexperiment/exp_spinbitmutex_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.spinbitmutex + +package goexperiment + +const SpinbitMutex = true +const SpinbitMutexInt = 1 diff --git a/src/internal/goexperiment/exp_swissmap_off.go b/src/internal/goexperiment/exp_swissmap_off.go new file mode 100644 index 00000000..2af40aa6 --- /dev/null +++ b/src/internal/goexperiment/exp_swissmap_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.swissmap + +package goexperiment + +const SwissMap = false +const SwissMapInt = 0 diff --git a/src/internal/goexperiment/exp_swissmap_on.go b/src/internal/goexperiment/exp_swissmap_on.go new file mode 100644 index 00000000..73be49b6 --- /dev/null +++ b/src/internal/goexperiment/exp_swissmap_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.swissmap + +package goexperiment + +const SwissMap = true +const SwissMapInt = 1 diff --git a/src/internal/goexperiment/exp_synchashtriemap_off.go b/src/internal/goexperiment/exp_synchashtriemap_off.go new file mode 100644 index 00000000..cab23aac --- /dev/null +++ b/src/internal/goexperiment/exp_synchashtriemap_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.synchashtriemap + +package goexperiment + +const SyncHashTrieMap = false +const SyncHashTrieMapInt = 0 diff --git a/src/internal/goexperiment/exp_synchashtriemap_on.go b/src/internal/goexperiment/exp_synchashtriemap_on.go new file mode 100644 index 00000000..87433ef4 --- /dev/null +++ b/src/internal/goexperiment/exp_synchashtriemap_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.synchashtriemap + +package goexperiment + +const SyncHashTrieMap = true +const SyncHashTrieMapInt = 1 diff --git a/src/internal/goexperiment/exp_synctest_off.go b/src/internal/goexperiment/exp_synctest_off.go new file mode 100644 index 00000000..fade13f8 --- /dev/null +++ b/src/internal/goexperiment/exp_synctest_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.synctest + +package goexperiment + +const Synctest = false +const SynctestInt = 0 diff --git a/src/internal/goexperiment/exp_synctest_on.go b/src/internal/goexperiment/exp_synctest_on.go new file mode 100644 index 00000000..9c44be72 --- /dev/null +++ b/src/internal/goexperiment/exp_synctest_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.synctest + +package goexperiment + +const Synctest = true +const SynctestInt = 1 diff --git a/src/internal/goexperiment/flags.go b/src/internal/goexperiment/flags.go index 3f9c5af6..948ed5c8 100644 --- a/src/internal/goexperiment/flags.go +++ b/src/internal/goexperiment/flags.go @@ -51,7 +51,7 @@ package goexperiment // tags, experiments use the strings.ToLower of their field name. // // For the baseline experimental configuration, see -// objabi.experimentBaseline. +// [internal/buildcfg.ParseGOEXPERIMENT]. // // If you change this struct definition, run "go generate". type Flags struct { @@ -113,6 +113,19 @@ type Flags struct { // AliasTypeParams enables type parameters for alias types. // Requires that gotypesalias=1 is set with GODEBUG. - // This flag will be removed with Go 1.24. + // This flag will be removed with Go 1.25. AliasTypeParams bool + + // SwissMap enables the SwissTable-based map implementation. + SwissMap bool + + // SpinbitMutex enables the new "spinbit" mutex implementation on supported + // platforms. See https://go.dev/issue/68578. + SpinbitMutex bool + + // SyncHashTrieMap enables the HashTrieMap sync.Map implementation. + SyncHashTrieMap bool + + // Synctest enables the testing/synctest package. + Synctest bool } diff --git a/src/internal/goos/gengoos.go b/src/internal/goos/gengoos.go index 37d9706d..e0d4d38e 100644 --- a/src/internal/goos/gengoos.go +++ b/src/internal/goos/gengoos.go @@ -17,11 +17,11 @@ import ( var gooses []string func main() { - data, err := os.ReadFile("../../go/build/syslist.go") + data, err := os.ReadFile("../../internal/syslist/syslist.go") if err != nil { log.Fatal(err) } - const goosPrefix = `var knownOS = map[string]bool{` + const goosPrefix = `var KnownOS = map[string]bool{` inGOOS := false for _, line := range strings.Split(string(data), "\n") { if strings.HasPrefix(line, goosPrefix) { diff --git a/src/internal/goversion/goversion.go b/src/internal/goversion/goversion.go index a9d6f12e..de2bcf4c 100644 --- a/src/internal/goversion/goversion.go +++ b/src/internal/goversion/goversion.go @@ -8,5 +8,5 @@ package goversion // in development and will eventually get released. // // It should be updated at the start of each development cycle to be -// the version of the next Go 1.x release. See golang.org/issue/40705. -const Version = 23 +// the version of the next Go 1.x release. See go.dev/issue/40705. +const Version = 24 diff --git a/src/internal/pkgbits/decoder.go b/src/internal/pkgbits/decoder.go index 4fe024d4..5a2b3f4a 100644 --- a/src/internal/pkgbits/decoder.go +++ b/src/internal/pkgbits/decoder.go @@ -21,7 +21,7 @@ import ( // export data. type PkgDecoder struct { // version is the file format version. - version uint32 + version Version // sync indicates whether the file uses sync markers. sync bool @@ -68,8 +68,6 @@ func (pr *PkgDecoder) SyncMarkers() bool { return pr.sync } // NewPkgDecoder returns a PkgDecoder initialized to read the Unified // IR export data from input. pkgPath is the package path for the // compilation unit that produced the export data. -// -// TODO(mdempsky): Remove pkgPath parameter; unneeded since CL 391014. func NewPkgDecoder(pkgPath, input string) PkgDecoder { pr := PkgDecoder{ pkgPath: pkgPath, @@ -80,14 +78,15 @@ func NewPkgDecoder(pkgPath, input string) PkgDecoder { r := strings.NewReader(input) - assert(binary.Read(r, binary.LittleEndian, &pr.version) == nil) + var ver uint32 + assert(binary.Read(r, binary.LittleEndian, &ver) == nil) + pr.version = Version(ver) - switch pr.version { - default: - panic(fmt.Errorf("unsupported version: %v", pr.version)) - case 0: - // no flags - case 1: + if pr.version >= numVersions { + panic(fmt.Errorf("cannot decode %q, export data version %d is greater than maximum supported version %d", pkgPath, pr.version, numVersions-1)) + } + + if pr.version.Has(Flags) { var flags uint32 assert(binary.Read(r, binary.LittleEndian, &flags) == nil) pr.sync = flags&flagSyncMarkers != 0 @@ -102,7 +101,9 @@ func NewPkgDecoder(pkgPath, input string) PkgDecoder { assert(err == nil) pr.elemData = input[pos:] - assert(len(pr.elemData)-8 == int(pr.elemEnds[len(pr.elemEnds)-1])) + + const fingerprintSize = 8 + assert(len(pr.elemData)-fingerprintSize == int(pr.elemEnds[len(pr.elemEnds)-1])) return pr } @@ -136,7 +137,7 @@ func (pr *PkgDecoder) AbsIdx(k RelocKind, idx Index) int { absIdx += int(pr.elemEndsEnds[k-1]) } if absIdx >= int(pr.elemEndsEnds[k]) { - errorf("%v:%v is out of bounds; %v", k, idx, pr.elemEndsEnds) + panicf("%v:%v is out of bounds; %v", k, idx, pr.elemEndsEnds) } return absIdx } @@ -242,7 +243,7 @@ type Decoder struct { func (r *Decoder) checkErr(err error) { if err != nil { - errorf("unexpected decoding error: %w", err) + panicf("unexpected decoding error: %w", err) } } @@ -371,7 +372,7 @@ func (r *Decoder) Int64() int64 { return r.rawVarint() } -// Int64 decodes and returns a uint64 value from the element bitstream. +// Uint64 decodes and returns a uint64 value from the element bitstream. func (r *Decoder) Uint64() uint64 { r.Sync(SyncUint64) return r.rawUvarint() @@ -513,3 +514,6 @@ func (pr *PkgDecoder) PeekObj(idx Index) (string, string, CodeObj) { return path, name, tag } + +// Version reports the version of the bitstream. +func (w *Decoder) Version() Version { return w.common.version } diff --git a/src/internal/pkgbits/encoder.go b/src/internal/pkgbits/encoder.go index 70a2cbae..015842f5 100644 --- a/src/internal/pkgbits/encoder.go +++ b/src/internal/pkgbits/encoder.go @@ -6,7 +6,7 @@ package pkgbits import ( "bytes" - "crypto/md5" + "crypto/sha256" "encoding/binary" "go/constant" "io" @@ -15,20 +15,12 @@ import ( "strings" ) -// currentVersion is the current version number. -// -// - v0: initial prototype -// -// - v1: adds the flags uint32 word -// -// TODO(mdempsky): For the next version bump: -// - remove the legacy "has init" bool from the public root -// - remove obj's "derived func instance" bool -const currentVersion uint32 = 1 - // A PkgEncoder provides methods for encoding a package's Unified IR // export data. type PkgEncoder struct { + // version of the bitstream. + version Version + // elems holds the bitstream for previously encoded elements. elems [numRelocs][]string @@ -52,8 +44,9 @@ func (pw *PkgEncoder) SyncMarkers() bool { return pw.syncFrames >= 0 } // export data files, but can help diagnosing desync errors in // higher-level Unified IR reader/writer code. If syncFrames is // negative, then sync markers are omitted entirely. -func NewPkgEncoder(syncFrames int) PkgEncoder { +func NewPkgEncoder(version Version, syncFrames int) PkgEncoder { return PkgEncoder{ + version: version, stringsIdx: make(map[string]Index), syncFrames: syncFrames, } @@ -62,20 +55,22 @@ func NewPkgEncoder(syncFrames int) PkgEncoder { // DumpTo writes the package's encoded data to out0 and returns the // package fingerprint. func (pw *PkgEncoder) DumpTo(out0 io.Writer) (fingerprint [8]byte) { - h := md5.New() + h := sha256.New() out := io.MultiWriter(out0, h) writeUint32 := func(x uint32) { assert(binary.Write(out, binary.LittleEndian, x) == nil) } - writeUint32(currentVersion) + writeUint32(uint32(pw.version)) - var flags uint32 - if pw.SyncMarkers() { - flags |= flagSyncMarkers + if pw.version.Has(Flags) { + var flags uint32 + if pw.SyncMarkers() { + flags |= flagSyncMarkers + } + writeUint32(flags) } - writeUint32(flags) // Write elemEndsEnds. var sum uint32 @@ -194,7 +189,7 @@ func (w *Encoder) Flush() Index { func (w *Encoder) checkErr(err error) { if err != nil { - errorf("unexpected encoding error: %v", err) + panicf("unexpected encoding error: %v", err) } } @@ -298,7 +293,7 @@ func (w *Encoder) Len(x int) { assert(x >= 0); w.Uint64(uint64(x)) } // Int encodes and writes an int value into the element bitstream. func (w *Encoder) Int(x int) { w.Int64(int64(x)) } -// Len encodes and writes a uint value into the element bitstream. +// Uint encodes and writes a uint value into the element bitstream. func (w *Encoder) Uint(x uint) { w.Uint64(uint64(x)) } // Reloc encodes and writes a relocation for the given (section, @@ -359,7 +354,7 @@ func (w *Encoder) Value(val constant.Value) { func (w *Encoder) scalar(val constant.Value) { switch v := constant.Val(val).(type) { default: - errorf("unhandled %v (%v)", val, val.Kind()) + panicf("unhandled %v (%v)", val, val.Kind()) case bool: w.Code(ValBool) w.Bool(v) @@ -392,3 +387,6 @@ func (w *Encoder) bigFloat(v *big.Float) { b := v.Append(nil, 'p', -1) w.String(string(b)) // TODO: More efficient encoding. } + +// Version reports the version of the bitstream. +func (w *Encoder) Version() Version { return w.p.version } diff --git a/src/internal/pkgbits/pkgbits_test.go b/src/internal/pkgbits/pkgbits_test.go new file mode 100644 index 00000000..6f400474 --- /dev/null +++ b/src/internal/pkgbits/pkgbits_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgbits_test + +import ( + "internal/pkgbits" + "strings" + "testing" +) + +func TestRoundTrip(t *testing.T) { + for _, version := range []pkgbits.Version{ + pkgbits.V0, + pkgbits.V1, + pkgbits.V2, + } { + pw := pkgbits.NewPkgEncoder(version, -1) + w := pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPublic) + w.Flush() + + var b strings.Builder + _ = pw.DumpTo(&b) + input := b.String() + + pr := pkgbits.NewPkgDecoder("package_id", input) + r := pr.NewDecoder(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic) + + if r.Version() != w.Version() { + t.Errorf("Expected reader version %q to be the writer version %q", r.Version(), w.Version()) + } + } +} + +// Type checker to enforce that know V* have the constant values they must have. +var _ [0]bool = [pkgbits.V0]bool{} +var _ [1]bool = [pkgbits.V1]bool{} + +func TestVersions(t *testing.T) { + type vfpair struct { + v pkgbits.Version + f pkgbits.Field + } + + // has field tests + for _, c := range []vfpair{ + {pkgbits.V1, pkgbits.Flags}, + {pkgbits.V2, pkgbits.Flags}, + {pkgbits.V0, pkgbits.HasInit}, + {pkgbits.V1, pkgbits.HasInit}, + {pkgbits.V0, pkgbits.DerivedFuncInstance}, + {pkgbits.V1, pkgbits.DerivedFuncInstance}, + {pkgbits.V0, pkgbits.DerivedInfoNeeded}, + {pkgbits.V1, pkgbits.DerivedInfoNeeded}, + {pkgbits.V2, pkgbits.AliasTypeParamNames}, + } { + if !c.v.Has(c.f) { + t.Errorf("Expected version %v to have field %v", c.v, c.f) + } + } + + // does not have field tests + for _, c := range []vfpair{ + {pkgbits.V0, pkgbits.Flags}, + {pkgbits.V2, pkgbits.HasInit}, + {pkgbits.V2, pkgbits.DerivedFuncInstance}, + {pkgbits.V2, pkgbits.DerivedInfoNeeded}, + {pkgbits.V0, pkgbits.AliasTypeParamNames}, + {pkgbits.V1, pkgbits.AliasTypeParamNames}, + } { + if c.v.Has(c.f) { + t.Errorf("Expected version %v to not have field %v", c.v, c.f) + } + } +} diff --git a/src/internal/pkgbits/support.go b/src/internal/pkgbits/support.go index f7579dfd..50534a29 100644 --- a/src/internal/pkgbits/support.go +++ b/src/internal/pkgbits/support.go @@ -12,6 +12,6 @@ func assert(b bool) { } } -func errorf(format string, args ...any) { +func panicf(format string, args ...any) { panic(fmt.Errorf(format, args...)) } diff --git a/src/internal/pkgbits/version.go b/src/internal/pkgbits/version.go new file mode 100644 index 00000000..ba664f45 --- /dev/null +++ b/src/internal/pkgbits/version.go @@ -0,0 +1,85 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgbits + +// Version indicates a version of a unified IR bitstream. +// Each Version indicates the addition, removal, or change of +// new data in the bitstream. +// +// These are serialized to disk and the interpretation remains fixed. +type Version uint32 + +const ( + // V0: initial prototype. + // + // All data that is not assigned a Field is in version V0 + // and has not been deprecated. + V0 Version = iota + + // V1: adds the Flags uint32 word + V1 + + // V2: removes unused legacy fields and supports type parameters for aliases. + // - remove the legacy "has init" bool from the public root + // - remove obj's "derived func instance" bool + // - add a TypeParamNames field to ObjAlias + // - remove derived info "needed" bool + V2 + + numVersions = iota +) + +// Field denotes a unit of data in the serialized unified IR bitstream. +// It is conceptually a like field in a structure. +// +// We only really need Fields when the data may or may not be present +// in a stream based on the Version of the bitstream. +// +// Unlike much of pkgbits, Fields are not serialized and +// can change values as needed. +type Field int + +const ( + // Flags in a uint32 in the header of a bitstream + // that is used to indicate whether optional features are enabled. + Flags Field = iota + + // Deprecated: HasInit was a bool indicating whether a package + // has any init functions. + HasInit + + // Deprecated: DerivedFuncInstance was a bool indicating + // whether an object was a function instance. + DerivedFuncInstance + + // ObjAlias has a list of TypeParamNames. + AliasTypeParamNames + + // Deprecated: DerivedInfoNeeded was a bool indicating + // whether a type was a derived type. + DerivedInfoNeeded + + numFields = iota +) + +// introduced is the version a field was added. +var introduced = [numFields]Version{ + Flags: V1, + AliasTypeParamNames: V2, +} + +// removed is the version a field was removed in or 0 for fields +// that have not yet been deprecated. +// (So removed[f]-1 is the last version it is included in.) +var removed = [numFields]Version{ + HasInit: V2, + DerivedFuncInstance: V2, + DerivedInfoNeeded: V2, +} + +// Has reports whether field f is present in a bitstream at version v. +func (v Version) Has(f Field) bool { + return introduced[f] <= v && (v < removed[f] || removed[f] == V0) +} diff --git a/src/internal/platform/supported.go b/src/internal/platform/supported.go index a774247e..e864c37d 100644 --- a/src/internal/platform/supported.go +++ b/src/internal/platform/supported.go @@ -173,7 +173,8 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { "android/amd64", "android/arm", "android/arm64", "android/386", "freebsd/amd64", "darwin/amd64", "darwin/arm64", - "windows/amd64", "windows/386", "windows/arm64": + "windows/amd64", "windows/386", "windows/arm64", + "wasip1/wasm": return true } return false @@ -280,7 +281,7 @@ func FirstClass(goos, goarch string) bool { return distInfo[OSArch{goos, goarch}].FirstClass } -// Broken reportsr whether goos/goarch is considered a broken port. +// Broken reports whether goos/goarch is considered a broken port. // (See https://go.dev/wiki/PortingPolicy#broken-ports.) func Broken(goos, goarch string) bool { return distInfo[OSArch{goos, goarch}].Broken diff --git a/src/internal/platform/zosarch.go b/src/internal/platform/zosarch.go index 1df34851..ebde978a 100644 --- a/src/internal/platform/zosarch.go +++ b/src/internal/platform/zosarch.go @@ -111,6 +111,6 @@ var distInfo = map[OSArch]osArchInfo{ {"wasip1", "wasm"}: {}, {"windows", "386"}: {CgoSupported: true, FirstClass: true}, {"windows", "amd64"}: {CgoSupported: true, FirstClass: true}, - {"windows", "arm"}: {}, + {"windows", "arm"}: {Broken: true}, {"windows", "arm64"}: {CgoSupported: true}, } diff --git a/src/internal/poll/copy_file_range_freebsd.go b/src/internal/poll/copy_file_range_freebsd.go new file mode 100644 index 00000000..63fa013e --- /dev/null +++ b/src/internal/poll/copy_file_range_freebsd.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package poll + +import ( + "internal/syscall/unix" + "syscall" +) + +func supportCopyFileRange() bool { + return unix.SupportCopyFileRange() +} + +// For best performance, call copy_file_range() with the largest len value +// possible. It is interruptible on most file systems, so there is no penalty +// for using very large len values, even SSIZE_MAX. +const maxCopyFileRangeRound = 1<<31 - 1 + +func handleCopyFileRangeErr(err error, copied, written int64) (bool, error) { + switch err { + case syscall.ENOSYS: + // The copy_file_range(2) function first appeared in FreeBSD 13.0. + // Go supports FreeBSD >= 12, so the system call + // may not be present. We've detected the FreeBSD version with + // unix.SupportCopyFileRange() at the beginning of this function, + // but we still want to check for ENOSYS here to prevent some rare + // case like https://go.dev/issue/58592 + // + // If we see ENOSYS, we have certainly not transferred + // any data, so we can tell the caller that we + // couldn't handle the transfer and let them fall + // back to more generic code. + return false, nil + case syscall.EFBIG, syscall.EINVAL, syscall.EIO: + // For EFBIG, the copy has exceeds the process's file size limit + // or the maximum file size for the filesystem dst resides on, in + // this case, we leave it to generic copy. + // + // For EINVAL, there could be a few reasons: + // 1. Either dst or src refers to a file object that + // is not a regular file, for instance, a pipe. + // 2. src and dst refer to the same file and byte ranges + // overlap. + // 3. The flags argument is not 0. + // Neither of these cases should be considered handled by + // copy_file_range(2) because there is no data transfer, so + // just fall back to generic copy. + return false, nil + } + return true, err +} diff --git a/src/internal/poll/copy_file_range_linux.go b/src/internal/poll/copy_file_range_linux.go index 3d51333d..50b9f1bc 100644 --- a/src/internal/poll/copy_file_range_linux.go +++ b/src/internal/poll/copy_file_range_linux.go @@ -10,6 +10,10 @@ import ( "syscall" ) +func supportCopyFileRange() bool { + return isKernelVersionGE53() +} + var isKernelVersionGE53 = sync.OnceValue(func() bool { major, minor := unix.KernelVersion() // copy_file_range(2) is broken in various ways on kernels older than 5.3, @@ -18,104 +22,62 @@ var isKernelVersionGE53 = sync.OnceValue(func() bool { return major > 5 || (major == 5 && minor >= 3) }) -const maxCopyFileRangeRound = 1 << 30 +// For best performance, call copy_file_range() with the largest len value +// possible. Linux sets up a limitation of data transfer for most of its I/O +// system calls, as MAX_RW_COUNT (INT_MAX & PAGE_MASK). This value equals to +// the maximum integer value minus a page size that is typically 2^12=4096 bytes. +// That is to say, it's the maximum integer value with the lowest 12 bits unset, +// which is 0x7ffff000. +const maxCopyFileRangeRound = 0x7ffff000 -// CopyFileRange copies at most remain bytes of data from src to dst, using -// the copy_file_range system call. dst and src must refer to regular files. -func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) { - if !isKernelVersionGE53() { - return 0, false, nil - } - - for remain > 0 { - max := remain - if max > maxCopyFileRangeRound { - max = maxCopyFileRangeRound - } - n, err := copyFileRange(dst, src, int(max)) - switch err { - case syscall.ENOSYS: - // copy_file_range(2) was introduced in Linux 4.5. - // Go supports Linux >= 2.6.33, so the system call - // may not be present. - // - // If we see ENOSYS, we have certainly not transferred - // any data, so we can tell the caller that we - // couldn't handle the transfer and let them fall - // back to more generic code. - return 0, false, nil - case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM: - // Prior to Linux 5.3, it was not possible to - // copy_file_range across file systems. Similarly to - // the ENOSYS case above, if we see EXDEV, we have - // not transferred any data, and we can let the caller - // fall back to generic code. - // - // As for EINVAL, that is what we see if, for example, - // dst or src refer to a pipe rather than a regular - // file. This is another case where no data has been - // transferred, so we consider it unhandled. - // - // If src and dst are on CIFS, we can see EIO. - // See issue #42334. - // - // If the file is on NFS, we can see EOPNOTSUPP. - // See issue #40731. - // - // If the process is running inside a Docker container, - // we might see EPERM instead of ENOSYS. See issue - // #40893. Since EPERM might also be a legitimate error, - // don't mark copy_file_range(2) as unsupported. - return 0, false, nil - case nil: - if n == 0 { - // If we did not read any bytes at all, - // then this file may be in a file system - // where copy_file_range silently fails. - // https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0 - if written == 0 { - return 0, false, nil - } - // Otherwise src is at EOF, which means - // we are done. - return written, true, nil +func handleCopyFileRangeErr(err error, copied, written int64) (bool, error) { + switch err { + case syscall.ENOSYS: + // copy_file_range(2) was introduced in Linux 4.5. + // Go supports Linux >= 3.2, so the system call + // may not be present. + // + // If we see ENOSYS, we have certainly not transferred + // any data, so we can tell the caller that we + // couldn't handle the transfer and let them fall + // back to more generic code. + return false, nil + case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM: + // Prior to Linux 5.3, it was not possible to + // copy_file_range across file systems. Similarly to + // the ENOSYS case above, if we see EXDEV, we have + // not transferred any data, and we can let the caller + // fall back to generic code. + // + // As for EINVAL, that is what we see if, for example, + // dst or src refer to a pipe rather than a regular + // file. This is another case where no data has been + // transferred, so we consider it unhandled. + // + // If src and dst are on CIFS, we can see EIO. + // See issue #42334. + // + // If the file is on NFS, we can see EOPNOTSUPP. + // See issue #40731. + // + // If the process is running inside a Docker container, + // we might see EPERM instead of ENOSYS. See issue + // #40893. Since EPERM might also be a legitimate error, + // don't mark copy_file_range(2) as unsupported. + return false, nil + case nil: + if copied == 0 { + // If we did not read any bytes at all, + // then this file may be in a file system + // where copy_file_range silently fails. + // https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0 + if written == 0 { + return false, nil } - remain -= n - written += n - default: - return written, true, err - } - } - return written, true, nil -} -// copyFileRange performs one round of copy_file_range(2). -func copyFileRange(dst, src *FD, max int) (written int64, err error) { - // The signature of copy_file_range(2) is: - // - // ssize_t copy_file_range(int fd_in, loff_t *off_in, - // int fd_out, loff_t *off_out, - // size_t len, unsigned int flags); - // - // Note that in the call to unix.CopyFileRange below, we use nil - // values for off_in and off_out. For the system call, this means - // "use and update the file offsets". That is why we must acquire - // locks for both file descriptors (and why this whole machinery is - // in the internal/poll package to begin with). - if err := dst.writeLock(); err != nil { - return 0, err - } - defer dst.writeUnlock() - if err := src.readLock(); err != nil { - return 0, err - } - defer src.readUnlock() - var n int - for { - n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0) - if err != syscall.EINTR { - break + // Otherwise src is at EOF, which means + // we are done. } } - return int64(n), err + return true, err } diff --git a/src/internal/poll/copy_file_range_unix.go b/src/internal/poll/copy_file_range_unix.go new file mode 100644 index 00000000..d3d3aaee --- /dev/null +++ b/src/internal/poll/copy_file_range_unix.go @@ -0,0 +1,70 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build freebsd || linux + +package poll + +import "internal/syscall/unix" + +// CopyFileRange copies at most remain bytes of data from src to dst, using +// the copy_file_range system call. dst and src must refer to regular files. +func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) { + if !supportCopyFileRange() { + return 0, false, nil + } + + for remain > 0 { + max := remain + if max > maxCopyFileRangeRound { + max = maxCopyFileRangeRound + } + n, e := copyFileRange(dst, src, int(max)) + if n > 0 { + remain -= n + written += n + } + handled, err = handleCopyFileRangeErr(e, n, written) + if n == 0 || !handled || err != nil { + return + } + } + + return written, true, nil +} + +// copyFileRange performs one round of copy_file_range(2). +func copyFileRange(dst, src *FD, max int) (written int64, err error) { + // For Linux, the signature of copy_file_range(2) is: + // + // ssize_t copy_file_range(int fd_in, loff_t *off_in, + // int fd_out, loff_t *off_out, + // size_t len, unsigned int flags); + // + // For FreeBSD, the signature of copy_file_range(2) is: + // + // ssize_t + // copy_file_range(int infd, off_t *inoffp, int outfd, off_t *outoffp, + // size_t len, unsigned int flags); + // + // Note that in the call to unix.CopyFileRange below, we use nil + // values for off_in/off_out and inoffp/outoffp, which means "the file + // offset for infd(fd_in) or outfd(fd_out) respectively will be used and + // updated by the number of bytes copied". + // + // That is why we must acquire locks for both file descriptors (and why + // this whole machinery is in the internal/poll package to begin with). + if err := dst.writeLock(); err != nil { + return 0, err + } + defer dst.writeUnlock() + if err := src.readLock(); err != nil { + return 0, err + } + defer src.readUnlock() + return ignoringEINTR2(func() (int64, error) { + n, err := unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0) + return int64(n), err + }) +} diff --git a/src/internal/poll/fd_posix.go b/src/internal/poll/fd_posix.go index 5bd333b4..12f13864 100644 --- a/src/internal/poll/fd_posix.go +++ b/src/internal/poll/fd_posix.go @@ -77,3 +77,13 @@ func ignoringEINTR(fn func() error) error { } } } + +// ignoringEINTR2 is ignoringEINTR, but returning an additional value. +func ignoringEINTR2[T any](fn func() (T, error)) (T, error) { + for { + v, err := fn() + if err != syscall.EINTR { + return v, err + } + } +} diff --git a/src/internal/poll/fd_wasip1.go b/src/internal/poll/fd_wasip1.go index 195aaa95..4f83d287 100644 --- a/src/internal/poll/fd_wasip1.go +++ b/src/internal/poll/fd_wasip1.go @@ -225,11 +225,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.LeUint16(b)) + return uint64(byteorder.LEUint16(b)) case 4: - return uint64(byteorder.LeUint32(b)) + return uint64(byteorder.LEUint32(b)) case 8: - return uint64(byteorder.LeUint64(b)) + return uint64(byteorder.LEUint64(b)) default: panic("internal/poll: readInt with unsupported size") } diff --git a/src/internal/poll/sendfile_bsd.go b/src/internal/poll/sendfile_bsd.go deleted file mode 100644 index d1023d4e..00000000 --- a/src/internal/poll/sendfile_bsd.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd - -package poll - -import "syscall" - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// SendFile wraps the sendfile system call. -func SendFile(dstFD *FD, src int, pos, remain int64) (written int64, err error, handled bool) { - defer func() { - TestHookDidSendFile(dstFD, src, written, err, handled) - }() - if err := dstFD.writeLock(); err != nil { - return 0, err, false - } - defer dstFD.writeUnlock() - - if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { - return 0, err, false - } - - dst := dstFD.Sysfd - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - m := n - pos1 := pos - n, err = syscall.Sendfile(dst, src, &pos1, n) - if n > 0 { - pos += int64(n) - written += int64(n) - remain -= int64(n) - // (n, nil) indicates that sendfile(2) has transferred - // the exact number of bytes we requested, or some unretryable - // error have occurred with partial bytes sent. Either way, we - // don't need to go through the following logic to check EINTR - // or fell into dstFD.pd.waitWrite, just continue to send the - // next chunk or break the loop. - if n == m { - continue - } else if err != syscall.EAGAIN && - err != syscall.EINTR && - err != syscall.EBUSY { - // Particularly, EPIPE. Errors like that would normally lead - // the subsequent sendfile(2) call to (-1, EBADF). - break - } - } else if err != syscall.EAGAIN && err != syscall.EINTR { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile), and other errors. - // We should end the loop when there is no error - // returned from sendfile(2) or it is not a retryable error. - break - } - if err == syscall.EINTR { - continue - } - if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { - break - } - } - if err == syscall.EAGAIN { - err = nil - } - handled = written != 0 || (err != syscall.ENOSYS && err != syscall.EINVAL && err != syscall.EOPNOTSUPP && err != syscall.ENOTSUP) - return -} diff --git a/src/internal/poll/sendfile_linux.go b/src/internal/poll/sendfile_linux.go deleted file mode 100644 index 1c4130d4..00000000 --- a/src/internal/poll/sendfile_linux.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package poll - -import "syscall" - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// SendFile wraps the sendfile system call. -func SendFile(dstFD *FD, src int, remain int64) (written int64, err error, handled bool) { - defer func() { - TestHookDidSendFile(dstFD, src, written, err, handled) - }() - if err := dstFD.writeLock(); err != nil { - return 0, err, false - } - defer dstFD.writeUnlock() - - if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { - return 0, err, false - } - - dst := dstFD.Sysfd - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - n, err = syscall.Sendfile(dst, src, nil, n) - if n > 0 { - written += int64(n) - remain -= int64(n) - continue - } else if err != syscall.EAGAIN && err != syscall.EINTR { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile), and other errors. - // We should end the loop when there is no error - // returned from sendfile(2) or it is not a retryable error. - break - } - if err == syscall.EINTR { - continue - } - if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { - break - } - } - if err == syscall.EAGAIN { - err = nil - } - handled = written != 0 || (err != syscall.ENOSYS && err != syscall.EINVAL) - return -} diff --git a/src/internal/poll/sendfile_solaris.go b/src/internal/poll/sendfile_solaris.go index b7c3f81a..605323b9 100644 --- a/src/internal/poll/sendfile_solaris.go +++ b/src/internal/poll/sendfile_solaris.go @@ -4,66 +4,9 @@ package poll -import "syscall" +//go:cgo_ldflag "-lsendfile" // Not strictly needed, but very helpful for debugging, see issue #10221. // //go:cgo_import_dynamic _ _ "libsendfile.so" //go:cgo_import_dynamic _ _ "libsocket.so" - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// SendFile wraps the sendfile system call. -func SendFile(dstFD *FD, src int, pos, remain int64) (written int64, err error, handled bool) { - defer func() { - TestHookDidSendFile(dstFD, src, written, err, handled) - }() - if err := dstFD.writeLock(); err != nil { - return 0, err, false - } - defer dstFD.writeUnlock() - - if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { - return 0, err, false - } - - dst := dstFD.Sysfd - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - pos1 := pos - n, err = syscall.Sendfile(dst, src, &pos1, n) - if err == syscall.EAGAIN || err == syscall.EINTR { - // partial write may have occurred - n = int(pos1 - pos) - } - if n > 0 { - pos += int64(n) - written += int64(n) - remain -= int64(n) - continue - } else if err != syscall.EAGAIN && err != syscall.EINTR { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile), and other errors. - // We should end the loop when there is no error - // returned from sendfile(2) or it is not a retryable error. - break - } - if err == syscall.EINTR { - continue - } - if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { - break - } - } - if err == syscall.EAGAIN { - err = nil - } - handled = written != 0 || (err != syscall.ENOSYS && err != syscall.EINVAL) - return -} diff --git a/src/internal/poll/sendfile_unix.go b/src/internal/poll/sendfile_unix.go new file mode 100644 index 00000000..1105e056 --- /dev/null +++ b/src/internal/poll/sendfile_unix.go @@ -0,0 +1,170 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || linux || solaris + +package poll + +import ( + "io" + "runtime" + "syscall" +) + +// SendFile wraps the sendfile system call. +// +// It copies data from src (a file descriptor) to dstFD, +// starting at the current position of src. +// It updates the current position of src to after the +// copied data. +// +// If size is zero, it copies the rest of src. +// Otherwise, it copies up to size bytes. +// +// The handled return parameter indicates whether SendFile +// was able to handle some or all of the operation. +// If handled is false, sendfile was unable to perform the copy, +// has not modified the source or destination, +// and the caller should perform the copy using a fallback implementation. +func SendFile(dstFD *FD, src int, size int64) (n int64, err error, handled bool) { + if goos := runtime.GOOS; goos == "linux" || goos == "android" { + // Linux's sendfile doesn't require any setup: + // It sends from the current position of the source file and + // updates the position of the source after sending. + return sendFile(dstFD, src, nil, size) + } + + // Non-Linux sendfile implementations don't use the current position of the source file, + // so we need to look up the position, pass it explicitly, and adjust it after + // sendfile returns. + start, err := ignoringEINTR2(func() (int64, error) { + return syscall.Seek(src, 0, io.SeekCurrent) + }) + if err != nil { + return 0, err, false + } + + pos := start + n, err, handled = sendFile(dstFD, src, &pos, size) + if n > 0 { + ignoringEINTR2(func() (int64, error) { + return syscall.Seek(src, start+n, io.SeekStart) + }) + } + return n, err, handled +} + +// sendFile wraps the sendfile system call. +func sendFile(dstFD *FD, src int, offset *int64, size int64) (written int64, err error, handled bool) { + defer func() { + TestHookDidSendFile(dstFD, src, written, err, handled) + }() + if err := dstFD.writeLock(); err != nil { + return 0, err, false + } + defer dstFD.writeUnlock() + + if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { + return 0, err, false + } + + dst := dstFD.Sysfd + for { + // Some platforms support passing 0 to read to the end of the source, + // but all platforms support just writing a large value. + // + // Limit the maximum size to fit in an int32, to avoid any possible overflow. + chunk := 1<<31 - 1 + if size > 0 { + chunk = int(min(size-written, int64(chunk))) + } + var n int + n, err = sendFileChunk(dst, src, offset, chunk, written) + if n > 0 { + written += int64(n) + } + switch err { + case nil: + // We're done if sendfile copied no bytes + // (we're at the end of the source) + // or if we have a size limit and have reached it. + // + // If sendfile copied some bytes and we don't have a size limit, + // try again to see if there is more data to copy. + if n == 0 || (size > 0 && written >= size) { + return written, nil, true + } + case syscall.EAGAIN: + // *BSD and Darwin can return EAGAIN with n > 0, + // so check to see if the write has completed. + // So far as we know all other platforms only + // return EAGAIN when n == 0, but checking is harmless. + if size > 0 && written >= size { + return written, nil, true + } + if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { + return written, err, true + } + case syscall.EINTR: + // Retry. + case syscall.ENOSYS, syscall.EOPNOTSUPP, syscall.EINVAL: + // ENOSYS indicates no kernel support for sendfile. + // EINVAL indicates a FD type that does not support sendfile. + // + // On Linux, copy_file_range can return EOPNOTSUPP when copying + // to a NFS file (issue #40731); check for it here just in case. + return written, err, written > 0 + default: + // We want to handle ENOTSUP like EOPNOTSUPP. + // It's a pain to put it as a switch case + // because on Linux systems ENOTSUP == EOPNOTSUPP, + // so the compiler complains about a duplicate case. + if err == syscall.ENOTSUP { + return written, err, written > 0 + } + + // Not a retryable error. + return written, err, true + } + } +} + +func sendFileChunk(dst, src int, offset *int64, size int, written int64) (n int, err error) { + switch runtime.GOOS { + case "linux", "android": + // The offset is always nil on Linux. + n, err = syscall.Sendfile(dst, src, offset, size) + case "solaris", "illumos": + // Trust the offset, not the return value from sendfile. + start := *offset + n, err = syscall.Sendfile(dst, src, offset, size) + n = int(*offset - start) + // A quirk on Solaris/illumos: sendfile claims to support out_fd + // as a regular file but returns EINVAL when the out_fd + // is not a socket of SOCK_STREAM, while it actually sends + // out data anyway and updates the file offset. + // + // Another quirk: sendfile transfers data and returns EINVAL when being + // asked to transfer bytes more than the actual file size. For instance, + // the source file is wrapped in an io.LimitedReader with larger size + // than the actual file size. + // + // To handle these cases we ignore EINVAL if any call to sendfile was + // able to send data. + if err == syscall.EINVAL && (n > 0 || written > 0) { + err = nil + } + default: + start := *offset + n, err = syscall.Sendfile(dst, src, offset, size) + if n > 0 { + // The BSD implementations of syscall.Sendfile don't + // update the offset parameter (despite it being a *int64). + // + // Trust the return value from sendfile, not the offset. + *offset = start + int64(n) + } + } + return +} diff --git a/src/internal/poll/sock_cloexec.go b/src/internal/poll/sock_cloexec.go index cbf70218..466ee313 100644 --- a/src/internal/poll/sock_cloexec.go +++ b/src/internal/poll/sock_cloexec.go @@ -5,7 +5,7 @@ // This file implements accept for platforms that provide a fast path for // setting SetNonblock and CloseOnExec. -//go:build dragonfly || freebsd || (linux && !arm) || netbsd || openbsd +//go:build dragonfly || freebsd || linux || netbsd || openbsd package poll diff --git a/src/internal/poll/sock_cloexec_accept.go b/src/internal/poll/sock_cloexec_accept.go deleted file mode 100644 index 4b86de59..00000000 --- a/src/internal/poll/sock_cloexec_accept.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements accept for platforms that provide a fast path for -// setting SetNonblock and CloseOnExec, but don't necessarily have accept4. -// This is the code we used for accept in Go 1.17 and earlier. -// On Linux the accept4 system call was introduced in 2.6.28 kernel, -// and our minimum requirement is 2.6.32, so we simplified the function. -// Unfortunately, on ARM accept4 wasn't added until 2.6.36, so for ARM -// only we continue using the older code. - -//go:build linux && arm - -package poll - -import "syscall" - -// Wrapper around the accept system call that marks the returned file -// descriptor as nonblocking and close-on-exec. -func accept(s int) (int, syscall.Sockaddr, string, error) { - ns, sa, err := Accept4Func(s, syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC) - switch err { - case nil: - return ns, sa, "", nil - default: // errors other than the ones listed - return -1, sa, "accept4", err - case syscall.ENOSYS: // syscall missing - case syscall.EINVAL: // some Linux use this instead of ENOSYS - case syscall.EACCES: // some Linux use this instead of ENOSYS - case syscall.EFAULT: // some Linux use this instead of ENOSYS - } - - // See ../syscall/exec_unix.go for description of ForkLock. - // It is probably okay to hold the lock across syscall.Accept - // because we have put fd.sysfd into non-blocking mode. - // However, a call to the File method will put it back into - // blocking mode. We can't take that risk, so no use of ForkLock here. - ns, sa, err = AcceptFunc(s) - if err == nil { - syscall.CloseOnExec(ns) - } - if err != nil { - return -1, nil, "accept", err - } - if err = syscall.SetNonblock(ns, true); err != nil { - CloseFunc(ns) - return -1, nil, "setnonblock", err - } - return ns, sa, "", nil -} diff --git a/src/internal/profile/proto_test.go b/src/internal/profile/proto_test.go index 46c6d830..4c09f7c4 100644 --- a/src/internal/profile/proto_test.go +++ b/src/internal/profile/proto_test.go @@ -5,7 +5,7 @@ package profile import ( - "reflect" + "slices" "testing" ) @@ -34,7 +34,7 @@ func TestPackedEncoding(t *testing.T) { }, } { source := &packedInts{tc.uint64s, tc.int64s} - if got, want := marshal(source), tc.encoded; !reflect.DeepEqual(got, want) { + if got, want := marshal(source), tc.encoded; !slices.Equal(got, want) { t.Errorf("failed encode %d, got %v, want %v", i, got, want) } @@ -43,10 +43,10 @@ func TestPackedEncoding(t *testing.T) { t.Errorf("failed decode %d: %v", i, err) continue } - if got, want := dest.uint64s, tc.uint64s; !reflect.DeepEqual(got, want) { + if got, want := dest.uint64s, tc.uint64s; !slices.Equal(got, want) { t.Errorf("failed decode uint64s %d, got %v, want %v", i, got, want) } - if got, want := dest.int64s, tc.int64s; !reflect.DeepEqual(got, want) { + if got, want := dest.int64s, tc.int64s; !slices.Equal(got, want) { t.Errorf("failed decode int64s %d, got %v, want %v", i, got, want) } } diff --git a/src/internal/race/norace.go b/src/internal/race/norace.go index da650489..3fb00573 100644 --- a/src/internal/race/norace.go +++ b/src/internal/race/norace.go @@ -7,6 +7,7 @@ package race import ( + "internal/abi" "unsafe" ) @@ -30,9 +31,21 @@ func Enable() { func Read(addr unsafe.Pointer) { } +func ReadPC(addr unsafe.Pointer, callerpc, pc uintptr) { +} + +func ReadObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { +} + func Write(addr unsafe.Pointer) { } +func WritePC(addr unsafe.Pointer, callerpc, pc uintptr) { +} + +func WriteObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { +} + func ReadRange(addr unsafe.Pointer, len int) { } diff --git a/src/internal/race/race.go b/src/internal/race/race.go index d2c7e53e..bfcb24a2 100644 --- a/src/internal/race/race.go +++ b/src/internal/race/race.go @@ -7,48 +7,52 @@ package race import ( - "runtime" + "internal/abi" "unsafe" ) const Enabled = true -func Acquire(addr unsafe.Pointer) { - runtime.RaceAcquire(addr) -} +// Functions below pushed from runtime. -func Release(addr unsafe.Pointer) { - runtime.RaceRelease(addr) -} +//go:linkname Acquire +func Acquire(addr unsafe.Pointer) -func ReleaseMerge(addr unsafe.Pointer) { - runtime.RaceReleaseMerge(addr) -} +//go:linkname Release +func Release(addr unsafe.Pointer) -func Disable() { - runtime.RaceDisable() -} +//go:linkname ReleaseMerge +func ReleaseMerge(addr unsafe.Pointer) -func Enable() { - runtime.RaceEnable() -} +//go:linkname Disable +func Disable() -func Read(addr unsafe.Pointer) { - runtime.RaceRead(addr) -} +//go:linkname Enable +func Enable() -func Write(addr unsafe.Pointer) { - runtime.RaceWrite(addr) -} +//go:linkname Read +func Read(addr unsafe.Pointer) -func ReadRange(addr unsafe.Pointer, len int) { - runtime.RaceReadRange(addr, len) -} +//go:linkname ReadPC +func ReadPC(addr unsafe.Pointer, callerpc, pc uintptr) -func WriteRange(addr unsafe.Pointer, len int) { - runtime.RaceWriteRange(addr, len) -} +//go:linkname ReadObjectPC +func ReadObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) -func Errors() int { - return runtime.RaceErrors() -} +//go:linkname Write +func Write(addr unsafe.Pointer) + +//go:linkname WritePC +func WritePC(addr unsafe.Pointer, callerpc, pc uintptr) + +//go:linkname WriteObjectPC +func WriteObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) + +//go:linkname ReadRange +func ReadRange(addr unsafe.Pointer, len int) + +//go:linkname WriteRange +func WriteRange(addr unsafe.Pointer, len int) + +//go:linkname Errors +func Errors() int diff --git a/src/internal/runtime/atomic/atomic_386.go b/src/internal/runtime/atomic/atomic_386.go index a023badd..b6cdea61 100644 --- a/src/internal/runtime/atomic/atomic_386.go +++ b/src/internal/runtime/atomic/atomic_386.go @@ -53,6 +53,9 @@ func Xchg64(ptr *uint64, new uint64) uint64 //go:noescape func Xchg(ptr *uint32, new uint32) uint32 +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchguintptr(ptr *uintptr, new uintptr) uintptr diff --git a/src/internal/runtime/atomic/atomic_386.s b/src/internal/runtime/atomic/atomic_386.s index 08812c37..58a56e63 100644 --- a/src/internal/runtime/atomic/atomic_386.s +++ b/src/internal/runtime/atomic/atomic_386.s @@ -153,6 +153,14 @@ addloop: MOVL CX, ret_hi+16(FP) RET +// uint8 Xchg8(uint8 *ptr, uint8 new) +TEXT ·Xchg8(SB), NOSPLIT, $0-9 + MOVL ptr+0(FP), BX + MOVB new+4(FP), AX + XCHGB AX, 0(BX) + MOVB AX, ret+8(FP) + RET + TEXT ·Xchg(SB), NOSPLIT, $0-12 MOVL ptr+0(FP), BX MOVL new+4(FP), AX diff --git a/src/internal/runtime/atomic/atomic_amd64.go b/src/internal/runtime/atomic/atomic_amd64.go index b4399540..2a2d07e5 100644 --- a/src/internal/runtime/atomic/atomic_amd64.go +++ b/src/internal/runtime/atomic/atomic_amd64.go @@ -57,6 +57,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_amd64.s b/src/internal/runtime/atomic/atomic_amd64.s index ec75bf93..d6dc7a32 100644 --- a/src/internal/runtime/atomic/atomic_amd64.s +++ b/src/internal/runtime/atomic/atomic_amd64.s @@ -117,6 +117,18 @@ TEXT ·Xaddint64(SB), NOSPLIT, $0-24 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24 JMP ·Xadd64(SB) +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVQ ptr+0(FP), BX + MOVB new+8(FP), AX + XCHGB AX, 0(BX) + MOVB AX, ret+16(FP) + RET + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; diff --git a/src/internal/runtime/atomic/atomic_arm.go b/src/internal/runtime/atomic/atomic_arm.go index b58f643c..8d8ffcf7 100644 --- a/src/internal/runtime/atomic/atomic_arm.go +++ b/src/internal/runtime/atomic/atomic_arm.go @@ -74,6 +74,27 @@ func Xchg(addr *uint32, v uint32) uint32 { } } +//go:noescape +func Xchg8(addr *uint8, v uint8) uint8 + +//go:nosplit +func goXchg8(addr *uint8, v uint8) uint8 { + // Align down to 4 bytes and use 32-bit CAS. + addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3)) + shift := (uintptr(unsafe.Pointer(addr)) & 3) * 8 // little endian + word := uint32(v) << shift + mask := uint32(0xFF) << shift + + for { + old := *addr32 // Read the old 32-bit value + // Clear the old 8 bits then insert the new value + if Cas(addr32, old, (old&^mask)|word) { + // Return the old 8-bit value + return uint8((old & mask) >> shift) + } + } +} + //go:nosplit func Xchguintptr(addr *uintptr, v uintptr) uintptr { return uintptr(Xchg((*uint32)(unsafe.Pointer(addr)), uint32(v))) @@ -159,12 +180,14 @@ func goStore64(addr *uint64, v uint64) { addrLock(addr).unlock() } +//go:noescape +func Or8(addr *uint8, v uint8) + //go:nosplit -func Or8(addr *uint8, v uint8) { +func goOr8(addr *uint8, v uint8) { // Align down to 4 bytes and use 32-bit CAS. - uaddr := uintptr(unsafe.Pointer(addr)) - addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) - word := uint32(v) << ((uaddr & 3) * 8) // little endian + addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3)) + word := uint32(v) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian for { old := *addr32 if Cas(addr32, old, old|word) { @@ -173,13 +196,15 @@ func Or8(addr *uint8, v uint8) { } } +//go:noescape +func And8(addr *uint8, v uint8) + //go:nosplit -func And8(addr *uint8, v uint8) { +func goAnd8(addr *uint8, v uint8) { // Align down to 4 bytes and use 32-bit CAS. - uaddr := uintptr(unsafe.Pointer(addr)) - addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) - word := uint32(v) << ((uaddr & 3) * 8) // little endian - mask := uint32(0xFF) << ((uaddr & 3) * 8) // little endian + addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3)) + word := uint32(v) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian + mask := uint32(0xFF) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian word |= ^mask for { old := *addr32 diff --git a/src/internal/runtime/atomic/atomic_arm.s b/src/internal/runtime/atomic/atomic_arm.s index 1cf7d8f6..85cee049 100644 --- a/src/internal/runtime/atomic/atomic_arm.s +++ b/src/internal/runtime/atomic/atomic_arm.s @@ -228,6 +228,59 @@ store64loop: DMB MB_ISH RET +TEXT armAnd8<>(SB),NOSPLIT,$0-5 + // addr is already in R1 + MOVB v+4(FP), R2 + +and8loop: + LDREXB (R1), R6 + + DMB MB_ISHST + + AND R2, R6 + STREXB R6, (R1), R0 + CMP $0, R0 + BNE and8loop + + DMB MB_ISH + + RET + +TEXT armOr8<>(SB),NOSPLIT,$0-5 + // addr is already in R1 + MOVB v+4(FP), R2 + +or8loop: + LDREXB (R1), R6 + + DMB MB_ISHST + + ORR R2, R6 + STREXB R6, (R1), R0 + CMP $0, R0 + BNE or8loop + + DMB MB_ISH + + RET + +TEXT armXchg8<>(SB),NOSPLIT,$0-9 + // addr is already in R1 + MOVB v+4(FP), R2 +xchg8loop: + LDREXB (R1), R6 + + DMB MB_ISHST + + STREXB R2, (R1), R0 + CMP $0, R0 + BNE xchg8loop + + DMB MB_ISH + + MOVB R6, ret+8(FP) + RET + // The following functions all panic if their address argument isn't // 8-byte aligned. Since we're calling back into Go code to do this, // we have to cooperate with stack unwinding. In the normal case, the @@ -310,3 +363,45 @@ TEXT ·Store64(SB),NOSPLIT,$-4-12 JMP ·goStore64(SB) #endif JMP armStore64<>(SB) + +TEXT ·And8(SB),NOSPLIT,$-4-5 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + +// Uses STREXB/LDREXB that is armv6k or later. +// For simplicity we only enable this on armv7. +#ifndef GOARM_7 + MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11 + CMP $1, R11 + BEQ 2(PC) + JMP ·goAnd8(SB) +#endif + JMP armAnd8<>(SB) + +TEXT ·Or8(SB),NOSPLIT,$-4-5 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + +// Uses STREXB/LDREXB that is armv6k or later. +// For simplicity we only enable this on armv7. +#ifndef GOARM_7 + MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11 + CMP $1, R11 + BEQ 2(PC) + JMP ·goOr8(SB) +#endif + JMP armOr8<>(SB) + +TEXT ·Xchg8(SB),NOSPLIT,$-4-9 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + + // Uses STREXB/LDREXB that is armv6k or later. + // For simplicity we only enable this on armv7. +#ifndef GOARM_7 + MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11 + CMP $1, R11 + BEQ 2(PC) + JMP ·goXchg8(SB) +#endif + JMP armXchg8<>(SB) diff --git a/src/internal/runtime/atomic/atomic_arm64.go b/src/internal/runtime/atomic/atomic_arm64.go index c4c56ae8..f4aef193 100644 --- a/src/internal/runtime/atomic/atomic_arm64.go +++ b/src/internal/runtime/atomic/atomic_arm64.go @@ -24,6 +24,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_arm64.s b/src/internal/runtime/atomic/atomic_arm64.s index ede56538..09f3b53c 100644 --- a/src/internal/runtime/atomic/atomic_arm64.s +++ b/src/internal/runtime/atomic/atomic_arm64.s @@ -120,6 +120,30 @@ TEXT ·Store64(SB), NOSPLIT, $0-16 STLR R1, (R0) RET +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R0 + MOVB new+8(FP), R1 +#ifndef GOARM64_LSE + MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 + CBZ R4, load_store_loop +#endif + SWPALB R1, (R0), R2 + MOVB R2, ret+16(FP) + RET +#ifndef GOARM64_LSE +load_store_loop: + LDAXRB (R0), R2 + STLXRB R1, (R0), R3 + CBNZ R3, load_store_loop + MOVB R2, ret+16(FP) + RET +#endif + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; diff --git a/src/internal/runtime/atomic/atomic_loong64.go b/src/internal/runtime/atomic/atomic_loong64.go index de6d4b4b..1fa1a9fa 100644 --- a/src/internal/runtime/atomic/atomic_loong64.go +++ b/src/internal/runtime/atomic/atomic_loong64.go @@ -6,7 +6,15 @@ package atomic -import "unsafe" +import ( + "internal/cpu" + "unsafe" +) + +const ( + offsetLOONG64HasLAMCAS = unsafe.Offsetof(cpu.Loong64.HasLAMCAS) + offsetLoong64HasLAM_BH = unsafe.Offsetof(cpu.Loong64.HasLAM_BH) +) //go:noescape func Xadd(ptr *uint32, delta int32) uint32 @@ -17,6 +25,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_loong64.s b/src/internal/runtime/atomic/atomic_loong64.s index 1812cb95..5222b77e 100644 --- a/src/internal/runtime/atomic/atomic_loong64.s +++ b/src/internal/runtime/atomic/atomic_loong64.s @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include "go_asm.h" #include "textflag.h" // bool cas(uint32 *ptr, uint32 old, uint32 new) @@ -15,18 +16,32 @@ TEXT ·Cas(SB), NOSPLIT, $0-17 MOVV ptr+0(FP), R4 MOVW old+8(FP), R5 MOVW new+12(FP), R6 - DBAR + + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8 + BEQ R8, cas_again + MOVV R5, R7 // backup old value + AMCASDBW R6, (R4), R5 + BNE R7, R5, cas_fail0 + MOVV $1, R4 + MOVB R4, ret+16(FP) + RET +cas_fail0: + MOVB R0, ret+16(FP) + RET + + // Implemented using the ll-sc instruction pair + DBAR $0x14 // LoadAcquire barrier cas_again: MOVV R6, R7 LL (R4), R8 - BNE R5, R8, cas_fail + BNE R5, R8, cas_fail1 SC R7, (R4) BEQ R7, cas_again MOVV $1, R4 MOVB R4, ret+16(FP) - DBAR + DBAR $0x12 // StoreRelease barrier RET -cas_fail: +cas_fail1: MOVV $0, R4 JMP -4(PC) @@ -42,21 +57,41 @@ TEXT ·Cas64(SB), NOSPLIT, $0-25 MOVV ptr+0(FP), R4 MOVV old+8(FP), R5 MOVV new+16(FP), R6 - DBAR + + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8 + BEQ R8, cas64_again + MOVV R5, R7 // backup old value + AMCASDBV R6, (R4), R5 + BNE R7, R5, cas64_fail0 + MOVV $1, R4 + MOVB R4, ret+24(FP) + RET +cas64_fail0: + MOVB R0, ret+24(FP) + RET + + // Implemented using the ll-sc instruction pair + DBAR $0x14 cas64_again: MOVV R6, R7 LLV (R4), R8 - BNE R5, R8, cas64_fail + BNE R5, R8, cas64_fail1 SCV R7, (R4) BEQ R7, cas64_again MOVV $1, R4 MOVB R4, ret+24(FP) - DBAR + DBAR $0x12 RET -cas64_fail: +cas64_fail1: MOVV $0, R4 JMP -4(PC) +TEXT ·Casint32(SB),NOSPLIT,$0-17 + JMP ·Cas(SB) + +TEXT ·Casint64(SB),NOSPLIT,$0-25 + JMP ·Cas64(SB) + TEXT ·Casuintptr(SB), NOSPLIT, $0-25 JMP ·Cas64(SB) @@ -78,6 +113,9 @@ TEXT ·Xadduintptr(SB), NOSPLIT, $0-24 TEXT ·Loadint64(SB), NOSPLIT, $0-16 JMP ·Load64(SB) +TEXT ·Xaddint32(SB),NOSPLIT,$0-20 + JMP ·Xadd(SB) + TEXT ·Xaddint64(SB), NOSPLIT, $0-24 JMP ·Xadd64(SB) @@ -91,65 +129,92 @@ TEXT ·Xaddint64(SB), NOSPLIT, $0-24 TEXT ·Casp1(SB), NOSPLIT, $0-25 JMP ·Cas64(SB) -// uint32 xadd(uint32 volatile *ptr, int32 delta) +// uint32 Xadd(uint32 volatile *ptr, int32 delta) // Atomically: // *val += delta; // return *val; TEXT ·Xadd(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW delta+8(FP), R5 - DBAR - LL (R4), R6 - ADDU R6, R5, R7 - MOVV R7, R6 - SC R7, (R4) - BEQ R7, -4(PC) - MOVW R6, ret+16(FP) - DBAR + AMADDDBW R5, (R4), R6 + ADDV R6, R5, R4 + MOVW R4, ret+16(FP) RET +// func Xadd64(ptr *uint64, delta int64) uint64 TEXT ·Xadd64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV delta+8(FP), R5 - DBAR - LLV (R4), R6 - ADDVU R6, R5, R7 - MOVV R7, R6 - SCV R7, (R4) - BEQ R7, -4(PC) - MOVV R6, ret+16(FP) - DBAR + AMADDDBV R5, (R4), R6 + ADDV R6, R5, R4 + MOVV R4, ret+16(FP) RET +// uint8 Xchg8(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVV ptr+0(FP), R4 + MOVBU new+8(FP), R5 + + // R6 = ((ptr & 3) * 8) + AND $3, R4, R6 + SLLV $3, R6 + + // R7 = ((0xFF) << R6) ^ (-1) + MOVV $0xFF, R8 + SLLV R6, R8, R7 + XOR $-1, R7 + + // R4 = ptr & (~3) + MOVV $~3, R8 + AND R8, R4 + + // R5 = ((val) << R6) + SLLV R6, R5 + + DBAR $0x14 // LoadAcquire barrier +_xchg8_again: + LL (R4), R8 + MOVV R8, R9 // backup old val + AND R7, R8 + OR R5, R8 + SC R8, (R4) + BEQ R8, _xchg8_again + DBAR $0x12 // StoreRelease barrier + SRLV R6, R9, R9 + MOVBU R9, ret+16(FP) + RET + +// func Xchg(ptr *uint32, new uint32) uint32 TEXT ·Xchg(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW new+8(FP), R5 - - DBAR - MOVV R5, R6 - LL (R4), R7 - SC R6, (R4) - BEQ R6, -3(PC) - MOVW R7, ret+16(FP) - DBAR + AMSWAPDBW R5, (R4), R6 + MOVW R6, ret+16(FP) RET +// func Xchg64(ptr *uint64, new uint64) uint64 TEXT ·Xchg64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV new+8(FP), R5 - - DBAR - MOVV R5, R6 - LLV (R4), R7 - SCV R6, (R4) - BEQ R6, -3(PC) - MOVV R7, ret+16(FP) - DBAR + AMSWAPDBV R5, (R4), R6 + MOVV R6, ret+16(FP) RET TEXT ·Xchguintptr(SB), NOSPLIT, $0-24 JMP ·Xchg64(SB) +// func Xchgint32(ptr *int32, new int32) int32 +TEXT ·Xchgint32(SB), NOSPLIT, $0-20 + JMP ·Xchg(SB) + +// func Xchgint64(ptr *int64, new int64) int64 +TEXT ·Xchgint64(SB), NOSPLIT, $0-24 + JMP ·Xchg64(SB) + TEXT ·StorepNoWB(SB), NOSPLIT, $0-16 JMP ·Store64(SB) @@ -165,147 +230,105 @@ TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16 TEXT ·Store(SB), NOSPLIT, $0-12 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - MOVW R5, 0(R4) - DBAR + AMSWAPDBW R5, (R4), R0 RET TEXT ·Store8(SB), NOSPLIT, $0-9 MOVV ptr+0(FP), R4 MOVB val+8(FP), R5 - DBAR + MOVBU internal∕cpu·Loong64+const_offsetLoong64HasLAM_BH(SB), R6 + BEQ R6, _legacy_store8_ + AMSWAPDBB R5, (R4), R0 + RET +_legacy_store8_: + // StoreRelease barrier + DBAR $0x12 MOVB R5, 0(R4) - DBAR + DBAR $0x18 RET TEXT ·Store64(SB), NOSPLIT, $0-16 MOVV ptr+0(FP), R4 MOVV val+8(FP), R5 - DBAR - MOVV R5, 0(R4) - DBAR + AMSWAPDBV R5, (R4), R0 RET // void Or8(byte volatile*, byte); TEXT ·Or8(SB), NOSPLIT, $0-9 MOVV ptr+0(FP), R4 MOVBU val+8(FP), R5 - // Align ptr down to 4 bytes so we can use 32-bit load/store. + // R6 = ptr & (~3) MOVV $~3, R6 AND R4, R6 // R7 = ((ptr & 3) * 8) AND $3, R4, R7 SLLV $3, R7 - // Shift val for aligned ptr. R5 = val << R4 + // R5 = val << R7 SLLV R7, R5 - - DBAR - LL (R6), R7 - OR R5, R7 - SC R7, (R6) - BEQ R7, -4(PC) - DBAR + AMORDBW R5, (R6), R0 RET // void And8(byte volatile*, byte); TEXT ·And8(SB), NOSPLIT, $0-9 MOVV ptr+0(FP), R4 MOVBU val+8(FP), R5 - // Align ptr down to 4 bytes so we can use 32-bit load/store. + // R6 = ptr & (~3) MOVV $~3, R6 AND R4, R6 // R7 = ((ptr & 3) * 8) AND $3, R4, R7 SLLV $3, R7 - // Shift val for aligned ptr. R5 = val << R7 | ^(0xFF << R7) - MOVV $0xFF, R8 - SLLV R7, R5 - SLLV R7, R8 - NOR R0, R8 - OR R8, R5 - - DBAR - LL (R6), R7 - AND R5, R7 - SC R7, (R6) - BEQ R7, -4(PC) - DBAR + // R5 = ((val ^ 0xFF) << R7) ^ (-1) + XOR $255, R5 + SLLV R7, R5 + XOR $-1, R5 + AMANDDBW R5, (R6), R0 RET // func Or(addr *uint32, v uint32) TEXT ·Or(SB), NOSPLIT, $0-12 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - OR R5, R6 - SC R6, (R4) - BEQ R6, -4(PC) - DBAR + AMORDBW R5, (R4), R0 RET // func And(addr *uint32, v uint32) TEXT ·And(SB), NOSPLIT, $0-12 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - AND R5, R6 - SC R6, (R4) - BEQ R6, -4(PC) - DBAR + AMANDDBW R5, (R4), R0 RET // func Or32(addr *uint32, v uint32) old uint32 TEXT ·Or32(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - OR R5, R6, R7 - SC R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVW R6, ret+16(FP) + AMORDBW R5, (R4), R6 + MOVW R6, ret+16(FP) RET // func And32(addr *uint32, v uint32) old uint32 TEXT ·And32(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - AND R5, R6, R7 - SC R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVW R6, ret+16(FP) + AMANDDBW R5, (R4), R6 + MOVW R6, ret+16(FP) RET // func Or64(addr *uint64, v uint64) old uint64 TEXT ·Or64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV val+8(FP), R5 - DBAR - LLV (R4), R6 - OR R5, R6, R7 - SCV R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVV R6, ret+16(FP) + AMORDBV R5, (R4), R6 + MOVV R6, ret+16(FP) RET // func And64(addr *uint64, v uint64) old uint64 TEXT ·And64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV val+8(FP), R5 - DBAR - LLV (R4), R6 - AND R5, R6, R7 - SCV R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVV R6, ret+16(FP) + AMANDDBV R5, (R4), R6 + MOVV R6, ret+16(FP) RET // func Anduintptr(addr *uintptr, v uintptr) old uintptr @@ -319,38 +342,30 @@ TEXT ·Oruintptr(SB), NOSPLIT, $0-24 // uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr) TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12 MOVV ptr+0(FP), R19 - DBAR MOVWU 0(R19), R19 - DBAR + DBAR $0x14 // LoadAcquire barrier MOVW R19, ret+8(FP) RET // uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr) TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9 MOVV ptr+0(FP), R19 - DBAR MOVBU 0(R19), R19 - DBAR + DBAR $0x14 MOVB R19, ret+8(FP) RET // uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr) TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16 MOVV ptr+0(FP), R19 - DBAR MOVV 0(R19), R19 - DBAR + DBAR $0x14 MOVV R19, ret+8(FP) RET // void *internal∕runtime∕atomic·Loadp(void *volatile *ptr) TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16 - MOVV ptr+0(FP), R19 - DBAR - MOVV 0(R19), R19 - DBAR - MOVV R19, ret+8(FP) - RET + JMP ·Load64(SB) // uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr) TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12 diff --git a/src/internal/runtime/atomic/atomic_ppc64x.go b/src/internal/runtime/atomic/atomic_ppc64x.go index 33a92b53..590ba03e 100644 --- a/src/internal/runtime/atomic/atomic_ppc64x.go +++ b/src/internal/runtime/atomic/atomic_ppc64x.go @@ -17,6 +17,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_ppc64x.s b/src/internal/runtime/atomic/atomic_ppc64x.s index 75635b93..184a30c9 100644 --- a/src/internal/runtime/atomic/atomic_ppc64x.s +++ b/src/internal/runtime/atomic/atomic_ppc64x.s @@ -236,6 +236,22 @@ TEXT ·Xadd64(SB), NOSPLIT, $0-24 MOVD R3, ret+16(FP) RET +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R4 + MOVB new+8(FP), R5 + LWSYNC + LBAR (R4), R3 + STBCCC R5, (R4) + BNE -2(PC) + ISYNC + MOVB R3, ret+16(FP) + RET + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; diff --git a/src/internal/runtime/atomic/bench_test.go b/src/internal/runtime/atomic/bench_test.go index 798431cf..b5837c97 100644 --- a/src/internal/runtime/atomic/bench_test.go +++ b/src/internal/runtime/atomic/bench_test.go @@ -43,6 +43,22 @@ func BenchmarkAtomicStore(b *testing.B) { } } +func BenchmarkAtomicLoad8(b *testing.B) { + var x uint8 + sink = &x + for i := 0; i < b.N; i++ { + atomic.Load8(&x) + } +} + +func BenchmarkAtomicStore8(b *testing.B) { + var x uint8 + sink = &x + for i := 0; i < b.N; i++ { + atomic.Store8(&x, 0) + } +} + func BenchmarkAnd8(b *testing.B) { var x [512]uint8 // give byte its own cache line sink = &x diff --git a/src/internal/runtime/atomic/xchg8_test.go b/src/internal/runtime/atomic/xchg8_test.go new file mode 100644 index 00000000..016ce819 --- /dev/null +++ b/src/internal/runtime/atomic/xchg8_test.go @@ -0,0 +1,59 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 || arm || arm64 || loong64 || ppc64 || ppc64le + +package atomic_test + +import ( + "internal/runtime/atomic" + "testing" +) + +func TestXchg8(t *testing.T) { + var a [16]uint8 + for i := range a { + next := uint8(i + 50) + a[i] = next + } + b := a + + // Compare behavior against non-atomic implementation. Expect the operation + // to work at any byte offset and to not clobber neighboring values. + for i := range a { + next := uint8(i + 100) + pa := atomic.Xchg8(&a[i], next) + pb := b[i] + b[i] = next + if pa != pb { + t.Errorf("atomic.Xchg8(a[%d]); %d != %d", i, pa, pb) + } + if a != b { + t.Errorf("after atomic.Xchg8(a[%d]); %d != %d", i, a, b) + } + if t.Failed() { + break + } + } +} + +func BenchmarkXchg8(b *testing.B) { + var x [512]uint8 // give byte its own cache line + sink = &x + for i := 0; i < b.N; i++ { + atomic.Xchg8(&x[255], uint8(i)) + } +} + +func BenchmarkXchg8Parallel(b *testing.B) { + var x [512]uint8 // give byte its own cache line + sink = &x + b.RunParallel(func(pb *testing.PB) { + i := uint8(0) + for pb.Next() { + atomic.Xchg8(&x[255], i) + i++ + } + }) +} diff --git a/src/internal/runtime/maps/export_noswiss_test.go b/src/internal/runtime/maps/export_noswiss_test.go new file mode 100644 index 00000000..333fc6ce --- /dev/null +++ b/src/internal/runtime/maps/export_noswiss_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +// This file allows non-GOEXPERIMENT=swissmap builds (i.e., old map builds) to +// construct a swissmap table for running the tests in this package. + +package maps + +import ( + "internal/abi" + "unsafe" +) + +type instantiatedGroup[K comparable, V any] struct { + ctrls ctrlGroup + slots [abi.SwissMapGroupSlots]instantiatedSlot[K, V] +} + +type instantiatedSlot[K comparable, V any] struct { + key K + elem V +} + +func newTestMapType[K comparable, V any]() *abi.SwissMapType { + var m map[K]V + mTyp := abi.TypeOf(m) + omt := (*abi.OldMapType)(unsafe.Pointer(mTyp)) + + var grp instantiatedGroup[K, V] + var slot instantiatedSlot[K, V] + + mt := &abi.SwissMapType{ + Key: omt.Key, + Elem: omt.Elem, + Group: abi.TypeOf(grp), + Hasher: omt.Hasher, + SlotSize: unsafe.Sizeof(slot), + GroupSize: unsafe.Sizeof(grp), + ElemOff: unsafe.Offsetof(slot.elem), + } + if omt.NeedKeyUpdate() { + mt.Flags |= abi.SwissMapNeedKeyUpdate + } + if omt.HashMightPanic() { + mt.Flags |= abi.SwissMapHashMightPanic + } + return mt +} diff --git a/src/internal/runtime/maps/export_swiss_test.go b/src/internal/runtime/maps/export_swiss_test.go new file mode 100644 index 00000000..3c6faf5c --- /dev/null +++ b/src/internal/runtime/maps/export_swiss_test.go @@ -0,0 +1,19 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package maps + +import ( + "internal/abi" + "unsafe" +) + +func newTestMapType[K comparable, V any]() *abi.SwissMapType { + var m map[K]V + mTyp := abi.TypeOf(m) + mt := (*abi.SwissMapType)(unsafe.Pointer(mTyp)) + return mt +} diff --git a/src/internal/runtime/maps/export_test.go b/src/internal/runtime/maps/export_test.go new file mode 100644 index 00000000..2c7b05ea --- /dev/null +++ b/src/internal/runtime/maps/export_test.go @@ -0,0 +1,124 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "unsafe" +) + +type CtrlGroup = ctrlGroup + +const DebugLog = debugLog + +var AlignUpPow2 = alignUpPow2 + +const MaxTableCapacity = maxTableCapacity +const MaxAvgGroupLoad = maxAvgGroupLoad + +// This isn't equivalent to runtime.maxAlloc. It is fine for basic testing but +// we can't properly test hint alloc overflows with this. +const maxAllocTest = 1 << 30 + +func NewTestMap[K comparable, V any](hint uintptr) (*Map, *abi.SwissMapType) { + mt := newTestMapType[K, V]() + return NewMap(mt, hint, nil, maxAllocTest), mt +} + +func (m *Map) TableCount() int { + if m.dirLen <= 0 { + return 0 + } + return m.dirLen +} + +// Total group count, summed across all tables. +func (m *Map) GroupCount() uint64 { + if m.dirLen <= 0 { + if m.dirPtr == nil { + return 0 + } + return 1 + } + + var n uint64 + var lastTab *table + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + if t == lastTab { + continue + } + lastTab = t + n += t.groups.lengthMask + 1 + } + return n +} + +// Return a key from a group containing no empty slots. +// +// Returns nil if there are no full groups. +// Returns nil if a group is full but contains entirely deleted slots. +// Returns nil if the map is small. +func (m *Map) KeyFromFullGroup(typ *abi.SwissMapType) unsafe.Pointer { + if m.dirLen <= 0 { + return nil + } + + var lastTab *table + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + if t == lastTab { + continue + } + lastTab = t + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + match := g.ctrls().matchEmpty() + if match != 0 { + continue + } + + // All full or deleted slots. + for j := uintptr(0); j < abi.SwissMapGroupSlots; j++ { + if g.ctrls().get(j) == ctrlDeleted { + continue + } + slotKey := g.key(typ, j) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + return slotKey + } + } + } + + return nil +} + +// Returns nil if the map is small. +func (m *Map) TableFor(typ *abi.SwissMapType, key unsafe.Pointer) *table { + if m.dirLen <= 0 { + return nil + } + + hash := typ.Hasher(key, m.seed) + idx := m.directoryIndex(hash) + return m.directoryAt(idx) +} + +func (t *table) GrowthLeft() uint64 { + return uint64(t.growthLeft) +} + +// Returns the start address of the groups array. +func (t *table) GroupsStart() unsafe.Pointer { + return t.groups.data +} + +// Returns the length of the groups array. +func (t *table) GroupsLength() uintptr { + return uintptr(t.groups.lengthMask + 1) +} diff --git a/src/internal/runtime/maps/fuzz_test.go b/src/internal/runtime/maps/fuzz_test.go new file mode 100644 index 00000000..f3f33773 --- /dev/null +++ b/src/internal/runtime/maps/fuzz_test.go @@ -0,0 +1,212 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package maps implements Go's builtin map type. +package maps_test + +import ( + "bytes" + "encoding/binary" + "fmt" + "internal/runtime/maps" + "reflect" + "testing" + "unsafe" +) + +// The input to FuzzTable is a binary-encoded array of fuzzCommand structs. +// +// Each fuzz call begins with an empty Map[uint16, uint32]. +// +// Each command is then executed on the map in sequence. Operations with +// output (e.g., Get) are verified against a reference map. +type fuzzCommand struct { + Op fuzzOp + + // Used for Get, Put, Delete. + Key uint16 + + // Used for Put. + Elem uint32 +} + +// Encoded size of fuzzCommand. +var fuzzCommandSize = binary.Size(fuzzCommand{}) + +type fuzzOp uint8 + +const ( + fuzzOpGet fuzzOp = iota + fuzzOpPut + fuzzOpDelete +) + +func encode(fc []fuzzCommand) []byte { + var buf bytes.Buffer + if err := binary.Write(&buf, binary.LittleEndian, fc); err != nil { + panic(fmt.Sprintf("error writing %v: %v", fc, err)) + } + return buf.Bytes() +} + +func decode(b []byte) []fuzzCommand { + // Round b down to a multiple of fuzzCommand size. i.e., ignore extra + // bytes of input. + entries := len(b) / fuzzCommandSize + usefulSize := entries * fuzzCommandSize + b = b[:usefulSize] + + fc := make([]fuzzCommand, entries) + buf := bytes.NewReader(b) + if err := binary.Read(buf, binary.LittleEndian, &fc); err != nil { + panic(fmt.Sprintf("error reading %v: %v", b, err)) + } + + return fc +} + +func TestEncodeDecode(t *testing.T) { + fc := []fuzzCommand{ + { + Op: fuzzOpPut, + Key: 123, + Elem: 456, + }, + { + Op: fuzzOpGet, + Key: 123, + }, + } + + b := encode(fc) + got := decode(b) + if !reflect.DeepEqual(fc, got) { + t.Errorf("encode-decode roundtrip got %+v want %+v", got, fc) + } + + // Extra trailing bytes ignored. + b = append(b, 42) + got = decode(b) + if !reflect.DeepEqual(fc, got) { + t.Errorf("encode-decode (extra byte) roundtrip got %+v want %+v", got, fc) + } +} + +func FuzzTable(f *testing.F) { + // All of the ops. + f.Add(encode([]fuzzCommand{ + { + Op: fuzzOpPut, + Key: 123, + Elem: 456, + }, + { + Op: fuzzOpDelete, + Key: 123, + }, + { + Op: fuzzOpGet, + Key: 123, + }, + })) + + // Add enough times to trigger grow. + f.Add(encode([]fuzzCommand{ + { + Op: fuzzOpPut, + Key: 1, + Elem: 101, + }, + { + Op: fuzzOpPut, + Key: 2, + Elem: 102, + }, + { + Op: fuzzOpPut, + Key: 3, + Elem: 103, + }, + { + Op: fuzzOpPut, + Key: 4, + Elem: 104, + }, + { + Op: fuzzOpPut, + Key: 5, + Elem: 105, + }, + { + Op: fuzzOpPut, + Key: 6, + Elem: 106, + }, + { + Op: fuzzOpPut, + Key: 7, + Elem: 107, + }, + { + Op: fuzzOpPut, + Key: 8, + Elem: 108, + }, + { + Op: fuzzOpGet, + Key: 1, + }, + { + Op: fuzzOpDelete, + Key: 2, + }, + { + Op: fuzzOpPut, + Key: 2, + Elem: 42, + }, + { + Op: fuzzOpGet, + Key: 2, + }, + })) + + f.Fuzz(func(t *testing.T, in []byte) { + fc := decode(in) + if len(fc) == 0 { + return + } + + m, typ := maps.NewTestMap[uint16, uint32](8) + ref := make(map[uint16]uint32) + for _, c := range fc { + switch c.Op { + case fuzzOpGet: + elemPtr, ok := m.Get(typ, unsafe.Pointer(&c.Key)) + refElem, refOK := ref[c.Key] + + if ok != refOK { + t.Errorf("Get(%d) got ok %v want ok %v", c.Key, ok, refOK) + } + if !ok { + continue + } + gotElem := *(*uint32)(elemPtr) + if gotElem != refElem { + t.Errorf("Get(%d) got %d want %d", c.Key, gotElem, refElem) + } + case fuzzOpPut: + m.Put(typ, unsafe.Pointer(&c.Key), unsafe.Pointer(&c.Elem)) + ref[c.Key] = c.Elem + case fuzzOpDelete: + m.Delete(typ, unsafe.Pointer(&c.Key)) + delete(ref, c.Key) + default: + // Just skip this command to keep the fuzzer + // less constrained. + continue + } + } + }) +} diff --git a/src/internal/runtime/maps/group.go b/src/internal/runtime/maps/group.go new file mode 100644 index 00000000..f3e9d4d1 --- /dev/null +++ b/src/internal/runtime/maps/group.go @@ -0,0 +1,324 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "internal/goarch" + "internal/runtime/sys" + "unsafe" +) + +const ( + // Maximum load factor prior to growing. + // + // 7/8 is the same load factor used by Abseil, but Abseil defaults to + // 16 slots per group, so they get two empty slots vs our one empty + // slot. We may want to reevaluate if this is best for us. + maxAvgGroupLoad = 7 + + ctrlEmpty ctrl = 0b10000000 + ctrlDeleted ctrl = 0b11111110 + + bitsetLSB = 0x0101010101010101 + bitsetMSB = 0x8080808080808080 + bitsetEmpty = bitsetLSB * uint64(ctrlEmpty) + bitsetDeleted = bitsetLSB * uint64(ctrlDeleted) +) + +// bitset represents a set of slots within a group. +// +// The underlying representation depends on GOARCH. +// +// On AMD64, bitset uses one bit per slot, where the bit is set if the slot is +// part of the set. All of the ctrlGroup.match* methods are replaced with +// intrinsics that return this packed representation. +// +// On other architectures, bitset uses one byte per slot, where each byte is +// either 0x80 if the slot is part of the set or 0x00 otherwise. This makes it +// convenient to calculate for an entire group at once using standard +// arithemetic instructions. +type bitset uint64 + +// first returns the relative index of the first control byte in the group that +// is in the set. +// +// Preconditions: b is not 0 (empty). +func (b bitset) first() uintptr { + return bitsetFirst(b) +} + +// Portable implementation of first. +// +// On AMD64, this is replaced with an intrisic that simply does +// TrailingZeros64. There is no need to shift as the bitset is packed. +func bitsetFirst(b bitset) uintptr { + return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 +} + +// removeFirst clears the first set bit (that is, resets the least significant +// set bit to 0). +func (b bitset) removeFirst() bitset { + return b & (b - 1) +} + +// removeBelow clears all set bits below slot i (non-inclusive). +func (b bitset) removeBelow(i uintptr) bitset { + return bitsetRemoveBelow(b, i) +} + +// Portable implementation of removeBelow. +// +// On AMD64, this is replaced with an intrisic that clears the lower i bits. +func bitsetRemoveBelow(b bitset, i uintptr) bitset { + // Clear all bits below slot i's byte. + mask := (uint64(1) << (8 * uint64(i))) - 1 + return b &^ bitset(mask) +} + +// lowestSet returns true if the bit is set for the lowest index in the bitset. +// +// This is intended for use with shiftOutLowest to loop over all entries in the +// bitset regardless of whether they are set. +func (b bitset) lowestSet() bool { + return bitsetLowestSet(b) +} + +// Portable implementation of lowestSet. +// +// On AMD64, this is replaced with an intrisic that checks the lowest bit. +func bitsetLowestSet(b bitset) bool { + return b&(1<<7) != 0 +} + +// shiftOutLowest shifts the lowest entry out of the bitset. Afterwards, the +// lowest entry in the bitset corresponds to the next slot. +func (b bitset) shiftOutLowest() bitset { + return bitsetShiftOutLowest(b) +} + +// Portable implementation of shiftOutLowest. +// +// On AMD64, this is replaced with an intrisic that shifts a single bit. +func bitsetShiftOutLowest(b bitset) bitset { + return b >> 8 +} + +// Each slot in the hash table has a control byte which can have one of three +// states: empty, deleted, and full. They have the following bit patterns: +// +// empty: 1 0 0 0 0 0 0 0 +// deleted: 1 1 1 1 1 1 1 0 +// full: 0 h h h h h h h // h represents the H1 hash bits +// +// TODO(prattmic): Consider inverting the top bit so that the zero value is empty. +type ctrl uint8 + +// ctrlGroup is a fixed size array of abi.SwissMapGroupSlots control bytes +// stored in a uint64. +type ctrlGroup uint64 + +// get returns the i-th control byte. +func (g *ctrlGroup) get(i uintptr) ctrl { + if goarch.BigEndian { + return *(*ctrl)(unsafe.Add(unsafe.Pointer(g), 7-i)) + } + return *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) +} + +// set sets the i-th control byte. +func (g *ctrlGroup) set(i uintptr, c ctrl) { + if goarch.BigEndian { + *(*ctrl)(unsafe.Add(unsafe.Pointer(g), 7-i)) = c + return + } + *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) = c +} + +// setEmpty sets all the control bytes to empty. +func (g *ctrlGroup) setEmpty() { + *g = ctrlGroup(bitsetEmpty) +} + +// matchH2 returns the set of slots which are full and for which the 7-bit hash +// matches the given value. May return false positives. +func (g ctrlGroup) matchH2(h uintptr) bitset { + return ctrlGroupMatchH2(g, h) +} + +// Portable implementation of matchH2. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed instrinsified return value. +func ctrlGroupMatchH2(g ctrlGroup, h uintptr) bitset { + // NB: This generic matching routine produces false positive matches when + // h is 2^N and the control bytes have a seq of 2^N followed by 2^N+1. For + // example: if ctrls==0x0302 and h=02, we'll compute v as 0x0100. When we + // subtract off 0x0101 the first 2 bytes we'll become 0xffff and both be + // considered matches of h. The false positive matches are not a problem, + // just a rare inefficiency. Note that they only occur if there is a real + // match and never occur on ctrlEmpty, or ctrlDeleted. The subsequent key + // comparisons ensure that there is no correctness issue. + v := uint64(g) ^ (bitsetLSB * uint64(h)) + return bitset(((v - bitsetLSB) &^ v) & bitsetMSB) +} + +// matchEmpty returns the set of slots in the group that are empty. +func (g ctrlGroup) matchEmpty() bitset { + return ctrlGroupMatchEmpty(g) +} + +// Portable implementation of matchEmpty. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed instrinsified return value. +func ctrlGroupMatchEmpty(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is empty iff bit 7 is set and bit 1 is not. We could select any + // of the other bits here (e.g. v << 1 would also work). + v := uint64(g) + return bitset((v &^ (v << 6)) & bitsetMSB) +} + +// matchEmptyOrDeleted returns the set of slots in the group that are empty or +// deleted. +func (g ctrlGroup) matchEmptyOrDeleted() bitset { + return ctrlGroupMatchEmptyOrDeleted(g) +} + +// Portable implementation of matchEmptyOrDeleted. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed instrinsified return value. +func ctrlGroupMatchEmptyOrDeleted(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is empty or deleted iff bit 7 is set. + v := uint64(g) + return bitset(v & bitsetMSB) +} + +// matchFull returns the set of slots in the group that are full. +func (g ctrlGroup) matchFull() bitset { + return ctrlGroupMatchFull(g) +} + +// Portable implementation of matchFull. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed instrinsified return value. +func ctrlGroupMatchFull(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is full iff bit 7 is unset. + v := uint64(g) + return bitset(^v & bitsetMSB) +} + +// groupReference is a wrapper type representing a single slot group stored at +// data. +// +// A group holds abi.SwissMapGroupSlots slots (key/elem pairs) plus their +// control word. +type groupReference struct { + // data points to the group, which is described by typ.Group and has + // layout: + // + // type group struct { + // ctrls ctrlGroup + // slots [abi.SwissMapGroupSlots]slot + // } + // + // type slot struct { + // key typ.Key + // elem typ.Elem + // } + data unsafe.Pointer // data *typ.Group +} + +const ( + ctrlGroupsSize = unsafe.Sizeof(ctrlGroup(0)) + groupSlotsOffset = ctrlGroupsSize +) + +// alignUp rounds n up to a multiple of a. a must be a power of 2. +func alignUp(n, a uintptr) uintptr { + return (n + a - 1) &^ (a - 1) +} + +// alignUpPow2 rounds n up to the next power of 2. +// +// Returns true if round up causes overflow. +func alignUpPow2(n uint64) (uint64, bool) { + if n == 0 { + return 0, false + } + v := (uint64(1) << sys.Len64(n-1)) + if v == 0 { + return 0, true + } + return v, false +} + +// ctrls returns the group control word. +func (g *groupReference) ctrls() *ctrlGroup { + return (*ctrlGroup)(g.data) +} + +// key returns a pointer to the key at index i. +func (g *groupReference) key(typ *abi.SwissMapType, i uintptr) unsafe.Pointer { + offset := groupSlotsOffset + i*typ.SlotSize + + return unsafe.Pointer(uintptr(g.data) + offset) +} + +// elem returns a pointer to the element at index i. +func (g *groupReference) elem(typ *abi.SwissMapType, i uintptr) unsafe.Pointer { + offset := groupSlotsOffset + i*typ.SlotSize + typ.ElemOff + + return unsafe.Pointer(uintptr(g.data) + offset) +} + +// groupsReference is a wrapper type describing an array of groups stored at +// data. +type groupsReference struct { + // data points to an array of groups. See groupReference above for the + // definition of group. + data unsafe.Pointer // data *[length]typ.Group + + // lengthMask is the number of groups in data minus one (note that + // length must be a power of two). This allows computing i%length + // quickly using bitwise AND. + lengthMask uint64 +} + +// newGroups allocates a new array of length groups. +// +// Length must be a power of two. +func newGroups(typ *abi.SwissMapType, length uint64) groupsReference { + return groupsReference{ + // TODO: make the length type the same throughout. + data: newarray(typ.Group, int(length)), + lengthMask: length - 1, + } +} + +// group returns the group at index i. +func (g *groupsReference) group(typ *abi.SwissMapType, i uint64) groupReference { + // TODO(prattmic): Do something here about truncation on cast to + // uintptr on 32-bit systems? + offset := uintptr(i) * typ.GroupSize + + return groupReference{ + data: unsafe.Pointer(uintptr(g.data) + offset), + } +} diff --git a/src/internal/runtime/maps/map.go b/src/internal/runtime/maps/map.go new file mode 100644 index 00000000..62463351 --- /dev/null +++ b/src/internal/runtime/maps/map.go @@ -0,0 +1,772 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package maps implements Go's builtin map type. +package maps + +import ( + "internal/abi" + "internal/goarch" + "internal/runtime/math" + "internal/runtime/sys" + "unsafe" +) + +// This package contains the implementation of Go's builtin map type. +// +// The map design is based on Abseil's "Swiss Table" map design +// (https://abseil.io/about/design/swisstables), with additional modifications +// to cover Go's additional requirements, discussed below. +// +// Terminology: +// - Slot: A storage location of a single key/element pair. +// - Group: A group of abi.SwissMapGroupSlots (8) slots, plus a control word. +// - Control word: An 8-byte word which denotes whether each slot is empty, +// deleted, or used. If a slot is used, its control byte also contains the +// lower 7 bits of the hash (H2). +// - H1: Upper 57 bits of a hash. +// - H2: Lower 7 bits of a hash. +// - Table: A complete "Swiss Table" hash table. A table consists of one or +// more groups for storage plus metadata to handle operation and determining +// when to grow. +// - Map: The top-level Map type consists of zero or more tables for storage. +// The upper bits of the hash select which table a key belongs to. +// - Directory: Array of the tables used by the map. +// +// At its core, the table design is similar to a traditional open-addressed +// hash table. Storage consists of an array of groups, which effectively means +// an array of key/elem slots with some control words interspersed. Lookup uses +// the hash to determine an initial group to check. If, due to collisions, this +// group contains no match, the probe sequence selects the next group to check +// (see below for more detail about the probe sequence). +// +// The key difference occurs within a group. In a standard open-addressed +// linear probed hash table, we would check each slot one at a time to find a +// match. A swiss table utilizes the extra control word to check all 8 slots in +// parallel. +// +// Each byte in the control word corresponds to one of the slots in the group. +// In each byte, 1 bit is used to indicate whether the slot is in use, or if it +// is empty/deleted. The other 7 bits contain the lower 7 bits of the hash for +// the key in that slot. See [ctrl] for the exact encoding. +// +// During lookup, we can use some clever bitwise manipulation to compare all 8 +// 7-bit hashes against the input hash in parallel (see [ctrlGroup.matchH2]). +// That is, we effectively perform 8 steps of probing in a single operation. +// With SIMD instructions, this could be extended to 16 slots with a 16-byte +// control word. +// +// Since we only use 7 bits of the 64 bit hash, there is a 1 in 128 (~0.7%) +// probability of false positive on each slot, but that's fine: we always need +// double check each match with a standard key comparison regardless. +// +// Probing +// +// Probing is done using the upper 57 bits (H1) of the hash as an index into +// the groups array. Probing walks through the groups using quadratic probing +// until it finds a group with a match or a group with an empty slot. See +// [probeSeq] for specifics about the probe sequence. Note the probe +// invariants: the number of groups must be a power of two, and the end of a +// probe sequence must be a group with an empty slot (the table can never be +// 100% full). +// +// Deletion +// +// Probing stops when it finds a group with an empty slot. This affects +// deletion: when deleting from a completely full group, we must not mark the +// slot as empty, as there could be more slots used later in a probe sequence +// and this deletion would cause probing to stop too early. Instead, we mark +// such slots as "deleted" with a tombstone. If the group still has an empty +// slot, we don't need a tombstone and directly mark the slot empty. Insert +// prioritizes reuse of tombstones over filling an empty slots. Otherwise, +// tombstones are only completely cleared during grow, as an in-place cleanup +// complicates iteration. +// +// Growth +// +// The probe sequence depends on the number of groups. Thus, when growing the +// group count all slots must be reordered to match the new probe sequence. In +// other words, an entire table must be grown at once. +// +// In order to support incremental growth, the map splits its contents across +// multiple tables. Each table is still a full hash table, but an individual +// table may only service a subset of the hash space. Growth occurs on +// individual tables, so while an entire table must grow at once, each of these +// grows is only a small portion of a map. The maximum size of a single grow is +// limited by limiting the maximum size of a table before it is split into +// multiple tables. +// +// A map starts with a single table. Up to [maxTableCapacity], growth simply +// replaces this table with a replacement with double capacity. Beyond this +// limit, growth splits the table into two. +// +// The map uses "extendible hashing" to select which table to use. In +// extendible hashing, we use the upper bits of the hash as an index into an +// array of tables (called the "directory"). The number of bits uses increases +// as the number of tables increases. For example, when there is only 1 table, +// we use 0 bits (no selection necessary). When there are 2 tables, we use 1 +// bit to select either the 0th or 1st table. [Map.globalDepth] is the number +// of bits currently used for table selection, and by extension (1 << +// globalDepth), the size of the directory. +// +// Note that each table has its own load factor and grows independently. If the +// 1st bucket grows, it will split. We'll need 2 bits to select tables, though +// we'll have 3 tables total rather than 4. We support this by allowing +// multiple indicies to point to the same table. This example: +// +// directory (globalDepth=2) +// +----+ +// | 00 | --\ +// +----+ +--> table (localDepth=1) +// | 01 | --/ +// +----+ +// | 10 | ------> table (localDepth=2) +// +----+ +// | 11 | ------> table (localDepth=2) +// +----+ +// +// Tables track the depth they were created at (localDepth). It is necessary to +// grow the directory when splitting a table where globalDepth == localDepth. +// +// Iteration +// +// Iteration is the most complex part of the map due to Go's generous iteration +// semantics. A summary of semantics from the spec: +// 1. Adding and/or deleting entries during iteration MUST NOT cause iteration +// to return the same entry more than once. +// 2. Entries added during iteration MAY be returned by iteration. +// 3. Entries modified during iteration MUST return their latest value. +// 4. Entries deleted during iteration MUST NOT be returned by iteration. +// 5. Iteration order is unspecified. In the implementation, it is explicitly +// randomized. +// +// If the map never grows, these semantics are straightforward: just iterate +// over every table in the directory and every group and slot in each table. +// These semantics all land as expected. +// +// If the map grows during iteration, things complicate significantly. First +// and foremost, we need to track which entries we already returned to satisfy +// (1). There are three types of grow: +// a. A table replaced by a single larger table. +// b. A table split into two replacement tables. +// c. Growing the directory (occurs as part of (b) if necessary). +// +// For all of these cases, the replacement table(s) will have a different probe +// sequence, so simply tracking the current group and slot indices is not +// sufficient. +// +// For (a) and (b), note that grows of tables other than the one we are +// currently iterating over are irrelevant. +// +// We handle (a) and (b) by having the iterator keep a reference to the table +// it is currently iterating over, even after the table is replaced. We keep +// iterating over the original table to maintain the iteration order and avoid +// violating (1). Any new entries added only to the replacement table(s) will +// be skipped (allowed by (2)). To avoid violating (3) or (4), while we use the +// original table to select the keys, we must look them up again in the new +// table(s) to determine if they have been modified or deleted. There is yet +// another layer of complexity if the key does not compare equal itself. See +// [Iter.Next] for the gory details. +// +// Note that for (b) once we finish iterating over the old table we'll need to +// skip the next entry in the directory, as that contains the second split of +// the old table. We can use the old table's localDepth to determine the next +// logical index to use. +// +// For (b), we must adjust the current directory index when the directory +// grows. This is more straightforward, as the directory orders remains the +// same after grow, so we just double the index if the directory size doubles. + +// Extracts the H1 portion of a hash: the 57 upper bits. +// TODO(prattmic): what about 32-bit systems? +func h1(h uintptr) uintptr { + return h >> 7 +} + +// Extracts the H2 portion of a hash: the 7 bits not used for h1. +// +// These are used as an occupied control byte. +func h2(h uintptr) uintptr { + return h & 0x7f +} + +type Map struct { + // The number of filled slots (i.e. the number of elements in all + // tables). Excludes deleted slots. + // Must be first (known by the compiler, for len() builtin). + used uint64 + + // seed is the hash seed, computed as a unique random number per map. + seed uintptr + + // The directory of tables. + // + // Normally dirPtr points to an array of table pointers + // + // dirPtr *[dirLen]*table + // + // The length (dirLen) of this array is `1 << globalDepth`. Multiple + // entries may point to the same table. See top-level comment for more + // details. + // + // Small map optimization: if the map always contained + // abi.SwissMapGroupSlots or fewer entries, it fits entirely in a + // single group. In that case dirPtr points directly to a single group. + // + // dirPtr *group + // + // In this case, dirLen is 0. used counts the number of used slots in + // the group. Note that small maps never have deleted slots (as there + // is no probe sequence to maintain). + dirPtr unsafe.Pointer + dirLen int + + // The number of bits to use in table directory lookups. + globalDepth uint8 + + // The number of bits to shift out of the hash for directory lookups. + // On 64-bit systems, this is 64 - globalDepth. + globalShift uint8 + + // writing is a flag that is toggled (XOR 1) while the map is being + // written. Normally it is set to 1 when writing, but if there are + // multiple concurrent writers, then toggling increases the probability + // that both sides will detect the race. + writing uint8 + + // clearSeq is a sequence counter of calls to Clear. It is used to + // detect map clears during iteration. + clearSeq uint64 +} + +func depthToShift(depth uint8) uint8 { + if goarch.PtrSize == 4 { + return 32 - depth + } + return 64 - depth +} + +// If m is non-nil, it should be used rather than allocating. +// +// maxAlloc should be runtime.maxAlloc. +// +// TODO(prattmic): Put maxAlloc somewhere accessible. +func NewMap(mt *abi.SwissMapType, hint uintptr, m *Map, maxAlloc uintptr) *Map { + if m == nil { + m = new(Map) + } + + m.seed = uintptr(rand()) + + if hint <= abi.SwissMapGroupSlots { + // A small map can fill all 8 slots, so no need to increase + // target capacity. + // + // In fact, since an 8 slot group is what the first assignment + // to an empty map would allocate anyway, it doesn't matter if + // we allocate here or on the first assignment. + // + // Thus we just return without allocating. (We'll save the + // allocation completely if no assignment comes.) + + // Note that the compiler may have initialized m.dirPtr with a + // pointer to a stack-allocated group, in which case we already + // have a group. The control word is already initialized. + + return m + } + + // Full size map. + + // Set initial capacity to hold hint entries without growing in the + // average case. + targetCapacity := (hint * abi.SwissMapGroupSlots) / maxAvgGroupLoad + if targetCapacity < hint { // overflow + return m // return an empty map. + } + + dirSize := (uint64(targetCapacity) + maxTableCapacity - 1) / maxTableCapacity + dirSize, overflow := alignUpPow2(dirSize) + if overflow || dirSize > uint64(math.MaxUintptr) { + return m // return an empty map. + } + + // Reject hints that are obviously too large. + groups, overflow := math.MulUintptr(uintptr(dirSize), maxTableCapacity) + if overflow { + return m // return an empty map. + } else { + mem, overflow := math.MulUintptr(groups, mt.GroupSize) + if overflow || mem > maxAlloc { + return m // return an empty map. + } + } + + m.globalDepth = uint8(sys.TrailingZeros64(dirSize)) + m.globalShift = depthToShift(m.globalDepth) + + directory := make([]*table, dirSize) + + for i := range directory { + // TODO: Think more about initial table capacity. + directory[i] = newTable(mt, uint64(targetCapacity)/dirSize, i, m.globalDepth) + } + + m.dirPtr = unsafe.Pointer(&directory[0]) + m.dirLen = len(directory) + + return m +} + +func NewEmptyMap() *Map { + m := new(Map) + m.seed = uintptr(rand()) + // See comment in NewMap. No need to eager allocate a group. + return m +} + +func (m *Map) directoryIndex(hash uintptr) uintptr { + if m.dirLen == 1 { + return 0 + } + return hash >> (m.globalShift & 63) +} + +func (m *Map) directoryAt(i uintptr) *table { + return *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) +} + +func (m *Map) directorySet(i uintptr, nt *table) { + *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) = nt +} + +func (m *Map) replaceTable(nt *table) { + // The number of entries that reference the same table doubles for each + // time the globalDepth grows without the table splitting. + entries := 1 << (m.globalDepth - nt.localDepth) + for i := 0; i < entries; i++ { + //m.directory[nt.index+i] = nt + m.directorySet(uintptr(nt.index+i), nt) + } +} + +func (m *Map) installTableSplit(old, left, right *table) { + if old.localDepth == m.globalDepth { + // No room for another level in the directory. Grow the + // directory. + newDir := make([]*table, m.dirLen*2) + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + newDir[2*i] = t + newDir[2*i+1] = t + // t may already exist in multiple indicies. We should + // only update t.index once. Since the index must + // increase, seeing the original index means this must + // be the first time we've encountered this table. + if t.index == i { + t.index = 2 * i + } + } + m.globalDepth++ + m.globalShift-- + //m.directory = newDir + m.dirPtr = unsafe.Pointer(&newDir[0]) + m.dirLen = len(newDir) + } + + // N.B. left and right may still consume multiple indicies if the + // directory has grown multiple times since old was last split. + left.index = old.index + m.replaceTable(left) + + entries := 1 << (m.globalDepth - left.localDepth) + right.index = left.index + entries + m.replaceTable(right) +} + +func (m *Map) Used() uint64 { + return m.used +} + +// Get performs a lookup of the key that key points to. It returns a pointer to +// the element, or false if the key doesn't exist. +func (m *Map) Get(typ *abi.SwissMapType, key unsafe.Pointer) (unsafe.Pointer, bool) { + return m.getWithoutKey(typ, key) +} + +func (m *Map) getWithKey(typ *abi.SwissMapType, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + if m.Used() == 0 { + return nil, nil, false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + return m.getWithKeySmall(typ, hash, key) + } + + idx := m.directoryIndex(hash) + return m.directoryAt(idx).getWithKey(typ, hash, key) +} + +func (m *Map) getWithoutKey(typ *abi.SwissMapType, key unsafe.Pointer) (unsafe.Pointer, bool) { + if m.Used() == 0 { + return nil, false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + return elem, ok + } + + idx := m.directoryIndex(hash) + return m.directoryAt(idx).getWithoutKey(typ, hash, key) +} + +func (m *Map) getWithKeySmall(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + g := groupReference{ + data: m.dirPtr, + } + + h2 := uint8(h2(hash)) + ctrls := *g.ctrls() + + for i := uintptr(0); i < abi.SwissMapGroupSlots; i++ { + c := uint8(ctrls) + ctrls >>= 8 + if c != h2 { + continue + } + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotKey, slotElem, true + } + } + + return nil, nil, false +} + +func (m *Map) Put(typ *abi.SwissMapType, key, elem unsafe.Pointer) { + slotElem := m.PutSlot(typ, key) + typedmemmove(typ.Elem, slotElem, elem) +} + +// PutSlot returns a pointer to the element slot where an inserted element +// should be written. +// +// PutSlot never returns nil. +func (m *Map) PutSlot(typ *abi.SwissMapType, key unsafe.Pointer) unsafe.Pointer { + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.SwissMapGroupSlots { + elem := m.putSlotSmall(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + // + // TODO(prattmic): If this is an update to an existing key then + // we actually don't need to grow. + m.growToTable(typ) + } + + for { + idx := m.directoryIndex(hash) + elem, ok := m.directoryAt(idx).PutSlot(typ, m, hash, key) + if !ok { + continue + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } +} + +func (m *Map) putSlotSmall(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + return nil + } + + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +func (m *Map) growToSmall(typ *abi.SwissMapType) { + grp := newGroups(typ, 1) + m.dirPtr = grp.data + + g := groupReference{ + data: m.dirPtr, + } + g.ctrls().setEmpty() +} + +func (m *Map) growToTable(typ *abi.SwissMapType) { + tab := newTable(typ, 2*abi.SwissMapGroupSlots, 0, 0) + + g := groupReference{ + data: m.dirPtr, + } + + for i := uintptr(0); i < abi.SwissMapGroupSlots; i++ { + if (g.ctrls().get(i) & ctrlEmpty) == ctrlEmpty { + // Empty + continue + } + + key := g.key(typ, i) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, i) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + + tab.uncheckedPutSlot(typ, hash, key, elem) + } + + directory := make([]*table, 1) + + directory[0] = tab + + m.dirPtr = unsafe.Pointer(&directory[0]) + m.dirLen = len(directory) + + m.globalDepth = 0 + m.globalShift = depthToShift(m.globalDepth) +} + +func (m *Map) Delete(typ *abi.SwissMapType, key unsafe.Pointer) { + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return + } + + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirLen == 0 { + m.deleteSmall(typ, hash, key) + } else { + idx := m.directoryIndex(hash) + m.directoryAt(idx).Delete(typ, m, hash, key) + } + + if m.used == 0 { + // Reset the hash seed to make it more difficult for attackers + // to repeatedly trigger hash collisions. See + // https://go.dev/issue/25237. + m.seed = uintptr(rand()) + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 +} + +func (m *Map) deleteSmall(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + slotKey := g.key(typ, i) + origSlotKey := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + m.used-- + + if typ.IndirectKey() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(origSlotKey) = nil + } else if typ.Key.Pointers() { + // Only bother clearing if there are pointers. + typedmemclr(typ.Key, slotKey) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(slotElem) = nil + } else { + // Unlike keys, always clear the elem (even if + // it contains no pointers), as compound + // assignment operations depend on cleared + // deleted values. See + // https://go.dev/issue/25936. + typedmemclr(typ.Elem, slotElem) + } + + // We only have 1 group, so it is OK to immediately + // reuse deleted slots. + g.ctrls().set(i, ctrlEmpty) + return + } + match = match.removeFirst() + } +} + +// Clear deletes all entries from the map resulting in an empty map. +func (m *Map) Clear(typ *abi.SwissMapType) { + if m == nil || m.Used() == 0 { + return + } + + if m.writing != 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 // toggle, see comment on writing + + if m.dirLen == 0 { + m.clearSmall(typ) + } else { + var lastTab *table + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + if t == lastTab { + continue + } + t.Clear(typ) + lastTab = t + } + m.used = 0 + m.clearSeq++ + // TODO: shrink directory? + } + + // Reset the hash seed to make it more difficult for attackers to + // repeatedly trigger hash collisions. See https://go.dev/issue/25237. + m.seed = uintptr(rand()) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 +} + +func (m *Map) clearSmall(typ *abi.SwissMapType) { + g := groupReference{ + data: m.dirPtr, + } + + typedmemclr(typ.Group, g.data) + g.ctrls().setEmpty() + + m.used = 0 + m.clearSeq++ +} diff --git a/src/internal/runtime/maps/map_swiss_test.go b/src/internal/runtime/maps/map_swiss_test.go new file mode 100644 index 00000000..6da00641 --- /dev/null +++ b/src/internal/runtime/maps/map_swiss_test.go @@ -0,0 +1,212 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests of map internals that need to use the builtin map type, and thus must +// be built with GOEXPERIMENT=swissmap. + +//go:build goexperiment.swissmap + +package maps_test + +import ( + "fmt" + "internal/abi" + "internal/runtime/maps" + "testing" + "unsafe" +) + +var alwaysFalse bool +var escapeSink any + +func escape[T any](x T) T { + if alwaysFalse { + escapeSink = x + } + return x +} + +const ( + belowMax = abi.SwissMapGroupSlots * 3 / 2 // 1.5 * group max = 2 groups @ 75% + atMax = (2 * abi.SwissMapGroupSlots * maps.MaxAvgGroupLoad) / abi.SwissMapGroupSlots // 2 groups at 7/8 full. +) + +func TestTableGroupCount(t *testing.T) { + // Test that maps of different sizes have the right number of + // tables/groups. + + type mapCount struct { + tables int + groups uint64 + } + + type mapCase struct { + initialLit mapCount + initialHint mapCount + after mapCount + } + + var testCases = []struct { + n int // n is the number of map elements + escape mapCase // expected values for escaping map + }{ + { + n: -(1 << 30), + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 0}, + }, + }, + { + n: -1, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 0}, + }, + }, + { + n: 0, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 0}, + }, + }, + { + n: 1, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 1}, + }, + }, + { + n: abi.SwissMapGroupSlots, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 1}, + }, + }, + { + n: abi.SwissMapGroupSlots + 1, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 2}, + after: mapCount{1, 2}, + }, + }, + { + n: belowMax, // 1.5 group max = 2 groups @ 75% + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 2}, + after: mapCount{1, 2}, + }, + }, + { + n: atMax, // 2 groups at max + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 2}, + after: mapCount{1, 2}, + }, + }, + { + n: atMax + 1, // 2 groups at max + 1 -> grow to 4 groups + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 4}, + after: mapCount{1, 4}, + }, + }, + { + n: 2 * belowMax, // 3 * group max = 4 groups @75% + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 4}, + after: mapCount{1, 4}, + }, + }, + { + n: 2*atMax + 1, // 4 groups at max + 1 -> grow to 8 groups + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 8}, + after: mapCount{1, 8}, + }, + }, + } + + testMap := func(t *testing.T, m map[int]int, n int, initial, after mapCount) { + mm := *(**maps.Map)(unsafe.Pointer(&m)) + + gotTab := mm.TableCount() + if gotTab != initial.tables { + t.Errorf("initial TableCount got %d want %d", gotTab, initial.tables) + } + + gotGroup := mm.GroupCount() + if gotGroup != initial.groups { + t.Errorf("initial GroupCount got %d want %d", gotGroup, initial.groups) + } + + for i := 0; i < n; i++ { + m[i] = i + } + + gotTab = mm.TableCount() + if gotTab != after.tables { + t.Errorf("after TableCount got %d want %d", gotTab, after.tables) + } + + gotGroup = mm.GroupCount() + if gotGroup != after.groups { + t.Errorf("after GroupCount got %d want %d", gotGroup, after.groups) + } + } + + t.Run("mapliteral", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(map[int]int{}) + testMap(t, m, tc.n, tc.escape.initialLit, tc.escape.after) + }) + }) + } + }) + t.Run("nohint", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(make(map[int]int)) + testMap(t, m, tc.n, tc.escape.initialLit, tc.escape.after) + }) + }) + } + }) + t.Run("makemap", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(make(map[int]int, tc.n)) + testMap(t, m, tc.n, tc.escape.initialHint, tc.escape.after) + }) + }) + } + }) + t.Run("makemap64", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(make(map[int]int, int64(tc.n))) + testMap(t, m, tc.n, tc.escape.initialHint, tc.escape.after) + }) + }) + } + }) +} diff --git a/src/internal/runtime/maps/map_test.go b/src/internal/runtime/maps/map_test.go new file mode 100644 index 00000000..160450eb --- /dev/null +++ b/src/internal/runtime/maps/map_test.go @@ -0,0 +1,701 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps_test + +import ( + "fmt" + "internal/abi" + "internal/runtime/maps" + "math" + "testing" + "unsafe" +) + +func TestCtrlSize(t *testing.T) { + cs := unsafe.Sizeof(maps.CtrlGroup(0)) + if cs != abi.SwissMapGroupSlots { + t.Errorf("ctrlGroup size got %d want abi.SwissMapGroupSlots %d", cs, abi.SwissMapGroupSlots) + } +} + +func TestMapPut(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + if m.Used() != 31 { + t.Errorf("Used() used got %d want 31", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*uint64)(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + } +} + +// Grow enough to cause a table split. +func TestMapSplit(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](0) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 2*maps.MaxTableCapacity; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + if m.Used() != 2*maps.MaxTableCapacity { + t.Errorf("Used() used got %d want 31", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 2*maps.MaxTableCapacity; i++ { + key += 1 + elem += 1 + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*uint64)(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + } +} + +func TestMapDelete(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](32) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + m.Delete(typ, unsafe.Pointer(&key)) + } + + if m.Used() != 0 { + t.Errorf("Used() used got %d want 0", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + _, ok := m.Get(typ, unsafe.Pointer(&key)) + if ok { + t.Errorf("Get(%d) got ok true want false", key) + } + } +} + +func TestTableClear(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](32) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + m.Clear(typ) + + if m.Used() != 0 { + t.Errorf("Clear() used got %d want 0", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + _, ok := m.Get(typ, unsafe.Pointer(&key)) + if ok { + t.Errorf("Get(%d) got ok true want false", key) + } + } +} + +// +0.0 and -0.0 compare equal, but we must still must update the key slot when +// overwriting. +func TestTableKeyUpdate(t *testing.T) { + m, typ := maps.NewTestMap[float64, uint64](8) + + zero := float64(0.0) + negZero := math.Copysign(zero, -1.0) + elem := uint64(0) + + m.Put(typ, unsafe.Pointer(&zero), unsafe.Pointer(&elem)) + if maps.DebugLog { + fmt.Printf("After put %f: %v\n", zero, m) + } + + elem = 1 + m.Put(typ, unsafe.Pointer(&negZero), unsafe.Pointer(&elem)) + if maps.DebugLog { + fmt.Printf("After put %f: %v\n", negZero, m) + } + + if m.Used() != 1 { + t.Errorf("Used() used got %d want 1", m.Used()) + } + + it := new(maps.Iter) + it.Init(typ, m) + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + t.Fatal("it.Key() got nil want key") + } + + key := *(*float64)(keyPtr) + elem = *(*uint64)(elemPtr) + if math.Copysign(1.0, key) > 0 { + t.Errorf("map key %f has positive sign", key) + } + if elem != 1 { + t.Errorf("map elem got %d want 1", elem) + } +} + +// Put should reuse a deleted slot rather than consuming an empty slot. +func TestTablePutDelete(t *testing.T) { + // Put will reuse the first deleted slot it encounters. + // + // This is awkward to test because Delete will only install ctrlDeleted + // if the group is full, otherwise it goes straight to empty. + // + // So first we must add to the table continuously until we happen to + // fill a group. + + // Avoid small maps, they have no tables. + m, typ := maps.NewTestMap[uint32, uint32](16) + + key := uint32(0) + elem := uint32(256 + 0) + + for { + key += 1 + elem += 1 + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + // Normally a Put that fills a group would fill it with the + // inserted key, so why search the whole map for a potentially + // different key in a full group? + // + // Put may grow/split a table. Initial construction of the new + // table(s) could result in a full group consisting of + // arbitrary keys. + fullKeyPtr := m.KeyFromFullGroup(typ) + if fullKeyPtr != nil { + // Found a full group. + key = *(*uint32)(fullKeyPtr) + elem = 256 + key + break + } + } + + // Key is in a full group. Deleting it will result in a ctrlDeleted + // slot. + m.Delete(typ, unsafe.Pointer(&key)) + + // Re-insert key. This should reuse the deleted slot rather than + // consuming space. + tabWant := m.TableFor(typ, unsafe.Pointer(&key)) + growthLeftWant := tabWant.GrowthLeft() + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + tabGot := m.TableFor(typ, unsafe.Pointer(&key)) + growthLeftGot := tabGot.GrowthLeft() + + if tabGot != tabWant { + // There shouldn't be a grow, as replacing a deleted slot + // doesn't require more space. + t.Errorf("Put(%d) grew table got %v want %v map %v", key, tabGot, tabWant, m) + } + + if growthLeftGot != growthLeftWant { + t.Errorf("GrowthLeft got %d want %d: map %v tab %v", growthLeftGot, growthLeftWant, m, tabGot) + } +} + +func TestTableIteration(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + + it := new(maps.Iter) + it.Init(typ, m) + for { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + got[key] = elem + } + + if len(got) != 31 { + t.Errorf("Iteration got %d entries, want 31: %+v", len(got), got) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + gotElem, ok := got[key] + if !ok { + t.Errorf("Iteration missing key %d", key) + continue + } + if gotElem != elem { + t.Errorf("Iteration key %d got elem %d want %d", key, gotElem, elem) + } + } +} + +// Deleted keys shouldn't be visible in iteration. +func TestTableIterationDelete(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + first := true + deletedKey := uint32(1) + it := new(maps.Iter) + it.Init(typ, m) + for { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + got[key] = elem + + if first { + first = false + + // If the key we intended to delete was the one we just + // saw, pick another to delete. + if key == deletedKey { + deletedKey++ + } + m.Delete(typ, unsafe.Pointer(&deletedKey)) + } + } + + if len(got) != 30 { + t.Errorf("Iteration got %d entries, want 30: %+v", len(got), got) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + + wantOK := true + if key == deletedKey { + wantOK = false + } + + gotElem, gotOK := got[key] + if gotOK != wantOK { + t.Errorf("Iteration key %d got ok %v want ok %v", key, gotOK, wantOK) + continue + } + if wantOK && gotElem != elem { + t.Errorf("Iteration key %d got elem %d want %d", key, gotElem, elem) + } + } +} + +// Deleted keys shouldn't be visible in iteration even after a grow. +func TestTableIterationGrowDelete(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + first := true + deletedKey := uint32(1) + it := new(maps.Iter) + it.Init(typ, m) + for { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + got[key] = elem + + if first { + first = false + + // If the key we intended to delete was the one we just + // saw, pick another to delete. + if key == deletedKey { + deletedKey++ + } + + // Double the number of elements to force a grow. + key := uint32(32) + elem := uint64(256 + 32) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + // Then delete from the grown map. + m.Delete(typ, unsafe.Pointer(&deletedKey)) + } + } + + // Don't check length: the number of new elements we'll see is + // unspecified. + + // Check values only of the original pre-iteration entries. + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + + wantOK := true + if key == deletedKey { + wantOK = false + } + + gotElem, gotOK := got[key] + if gotOK != wantOK { + t.Errorf("Iteration key %d got ok %v want ok %v", key, gotOK, wantOK) + continue + } + if wantOK && gotElem != elem { + t.Errorf("Iteration key %d got elem %d want %d", key, gotElem, elem) + } + } +} + +func testTableIterationGrowDuplicate(t *testing.T, grow int) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + it := new(maps.Iter) + it.Init(typ, m) + for i := 0; ; i++ { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + if elem != 256+uint64(key) { + t.Errorf("iteration got key %d elem %d want elem %d", key, elem, 256+uint64(key)) + } + if _, ok := got[key]; ok { + t.Errorf("iteration got key %d more than once", key) + } + got[key] = elem + + // Grow halfway through iteration. + if i == 16 { + key := uint32(32) + elem := uint64(256 + 32) + + for i := 0; i < grow; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + } + } + + // Don't check length: the number of new elements we'll see is + // unspecified. +} + +// Grow should not allow duplicate keys to appear. +func TestTableIterationGrowDuplicate(t *testing.T) { + // Small grow, only enough to cause table grow. + t.Run("grow", func(t *testing.T) { testTableIterationGrowDuplicate(t, 32) }) + + // Large grow, to cause table split. + t.Run("split", func(t *testing.T) { testTableIterationGrowDuplicate(t, 2*maps.MaxTableCapacity) }) +} + +func TestAlignUpPow2(t *testing.T) { + tests := []struct { + in uint64 + want uint64 + overflow bool + }{ + { + in: 0, + want: 0, + }, + { + in: 3, + want: 4, + }, + { + in: 4, + want: 4, + }, + { + in: 1 << 63, + want: 1 << 63, + }, + { + in: (1 << 63) - 1, + want: 1 << 63, + }, + { + in: (1 << 63) + 1, + overflow: true, + }, + } + + for _, tc := range tests { + got, overflow := maps.AlignUpPow2(tc.in) + if got != tc.want { + t.Errorf("alignUpPow2(%d) got %d, want %d", tc.in, got, tc.want) + } + if overflow != tc.overflow { + t.Errorf("alignUpPow2(%d) got overflow %v, want %v", tc.in, overflow, tc.overflow) + } + } +} + +// Verify that a map with zero-size slot is safe to use. +func TestMapZeroSizeSlot(t *testing.T) { + m, typ := maps.NewTestMap[struct{}, struct{}](16) + + key := struct{}{} + elem := struct{}{} + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*struct{})(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + + tab := m.TableFor(typ, unsafe.Pointer(&key)) + start := tab.GroupsStart() + length := tab.GroupsLength() + end := unsafe.Pointer(uintptr(start) + length*typ.GroupSize - 1) // inclusive to ensure we have a valid pointer + if uintptr(got) < uintptr(start) || uintptr(got) > uintptr(end) { + t.Errorf("elem address outside groups allocation; got %p want [%p, %p]", got, start, end) + } +} + +func TestMapIndirect(t *testing.T) { + type big [abi.SwissMapMaxKeyBytes + abi.SwissMapMaxElemBytes]byte + + m, typ := maps.NewTestMap[big, big](8) + + key := big{} + elem := big{} + elem[0] = 128 + + for i := 0; i < 31; i++ { + key[0] += 1 + elem[0] += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %v: %v\n", key, m) + } + } + + if m.Used() != 31 { + t.Errorf("Used() used got %d want 31", m.Used()) + } + + key = big{} + elem = big{} + elem[0] = 128 + + for i := 0; i < 31; i++ { + key[0] += 1 + elem[0] += 1 + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%v) got ok false want true", key) + } + gotElem := *(*big)(got) + if gotElem != elem { + t.Errorf("Get(%v) got elem %v want %v", key, gotElem, elem) + } + } +} + +// Delete should clear element. See https://go.dev/issue/25936. +func TestMapDeleteClear(t *testing.T) { + m, typ := maps.NewTestMap[int64, int64](8) + + key := int64(0) + elem := int64(128) + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*int64)(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + + m.Delete(typ, unsafe.Pointer(&key)) + + gotElem = *(*int64)(got) + if gotElem != 0 { + t.Errorf("Delete(%d) failed to clear element. got %d want 0", key, gotElem) + } +} diff --git a/src/internal/runtime/maps/runtime.go b/src/internal/runtime/maps/runtime.go new file mode 100644 index 00000000..3d06f54f --- /dev/null +++ b/src/internal/runtime/maps/runtime.go @@ -0,0 +1,30 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "unsafe" +) + +// Functions below pushed from runtime. + +//go:linkname fatal +func fatal(s string) + +//go:linkname rand +func rand() uint64 + +//go:linkname typedmemmove +func typedmemmove(typ *abi.Type, dst, src unsafe.Pointer) + +//go:linkname typedmemclr +func typedmemclr(typ *abi.Type, ptr unsafe.Pointer) + +//go:linkname newarray +func newarray(typ *abi.Type, n int) unsafe.Pointer + +//go:linkname newobject +func newobject(typ *abi.Type) unsafe.Pointer diff --git a/src/internal/runtime/maps/runtime_fast32_swiss.go b/src/internal/runtime/maps/runtime_fast32_swiss.go new file mode 100644 index 00000000..46023cc9 --- /dev/null +++ b/src/internal/runtime/maps/runtime_fast32_swiss.go @@ -0,0 +1,470 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package maps + +import ( + "internal/abi" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +//go:linkname runtime_mapaccess1_fast32 runtime.mapaccess1_fast32 +func runtime_mapaccess1_fast32(typ *abi.SwissMapType, m *Map, key uint32) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint32)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]) + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_fast32 runtime.mapaccess2_fast32 +func runtime_mapaccess2_fast32(typ *abi.SwissMapType, m *Map, key uint32) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint32)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem, true + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]), false + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFast32(typ *abi.SwissMapType, hash uintptr, key uint32) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*uint32)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_fast32 runtime.mapassign_fast32 +func runtime_mapassign_fast32(typ *abi.SwissMapType, m *Map, key uint32) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.SwissMapGroupSlots { + elem := m.putSlotSmallFast32(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*uint32)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +// Key is a 32-bit pointer (only called on 32-bit GOARCH). This source is identical to fast64ptr. +// +// TODO(prattmic): With some compiler refactoring we could avoid duplication of this function. +// +//go:linkname runtime_mapassign_fast32ptr runtime.mapassign_fast32ptr +func runtime_mapassign_fast32ptr(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.SwissMapGroupSlots { + elem := m.putSlotSmallFastPtr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot we + // find, which we'll use to insert the new entry if necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_fast32 runtime.mapdelete_fast32 +func runtime_mapdelete_fast32(typ *abi.SwissMapType, m *Map, key uint32) { + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/src/internal/runtime/maps/runtime_fast64_swiss.go b/src/internal/runtime/maps/runtime_fast64_swiss.go new file mode 100644 index 00000000..6bc6b2f0 --- /dev/null +++ b/src/internal/runtime/maps/runtime_fast64_swiss.go @@ -0,0 +1,509 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package maps + +import ( + "internal/abi" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +//go:linkname runtime_mapaccess1_fast64 runtime.mapaccess1_fast64 +func runtime_mapaccess1_fast64(typ *abi.SwissMapType, m *Map, key uint64) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint64)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]) + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_fast64 runtime.mapaccess2_fast64 +func runtime_mapaccess2_fast64(typ *abi.SwissMapType, m *Map, key uint64) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint64)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem, true + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]), false + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFast64(typ *abi.SwissMapType, hash uintptr, key uint64) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*uint64)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_fast64 runtime.mapassign_fast64 +func runtime_mapassign_fast64(typ *abi.SwissMapType, m *Map, key uint64) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.SwissMapGroupSlots { + elem := m.putSlotSmallFast64(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*uint64)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +func (m *Map) putSlotSmallFastPtr(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +// Key is a 64-bit pointer (only called on 64-bit GOARCH). +// +//go:linkname runtime_mapassign_fast64ptr runtime.mapassign_fast64ptr +func runtime_mapassign_fast64ptr(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.SwissMapGroupSlots { + elem := m.putSlotSmallFastPtr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_fast64 runtime.mapdelete_fast64 +func runtime_mapdelete_fast64(typ *abi.SwissMapType, m *Map, key uint64) { + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/src/internal/runtime/maps/runtime_faststr_swiss.go b/src/internal/runtime/maps/runtime_faststr_swiss.go new file mode 100644 index 00000000..077c05ae --- /dev/null +++ b/src/internal/runtime/maps/runtime_faststr_swiss.go @@ -0,0 +1,408 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package maps + +import ( + "internal/abi" + "internal/goarch" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +func (m *Map) getWithoutKeySmallFastStr(typ *abi.SwissMapType, key string) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + ctrls := *g.ctrls() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + + // The 64 threshold was chosen based on performance of BenchmarkMapStringKeysEight, + // where there are 8 keys to check, all of which don't quick-match the lookup key. + // In that case, we can save hashing the lookup key. That savings is worth this extra code + // for strings that are long enough that hashing is expensive. + if len(key) > 64 { + // String hashing and equality might be expensive. Do a quick check first. + j := abi.SwissMapGroupSlots + for i := range abi.SwissMapGroupSlots { + if ctrls&(1<<7) == 0 && longStringQuickEqualityTest(key, *(*string)(slotKey)) { + if j < abi.SwissMapGroupSlots { + // 2 strings both passed the quick equality test. + // Break out of this loop and do it the slow way. + goto dohash + } + j = i + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + ctrls >>= 8 + } + if j == abi.SwissMapGroupSlots { + // No slot passed the quick test. + return nil + } + // There's exactly one slot that passed the quick test. Do the single expensive comparison. + slotKey = g.key(typ, uintptr(j)) + if key == *(*string)(slotKey) { + return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + } + return nil + } + +dohash: + // This path will cost 1 hash and 1+ε comparisons. + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&key)), m.seed) + h2 := uint8(h2(hash)) + ctrls = *g.ctrls() + slotKey = g.key(typ, 0) + + for range abi.SwissMapGroupSlots { + if uint8(ctrls) == h2 && key == *(*string)(slotKey) { + return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + ctrls >>= 8 + } + return nil +} + +// Returns true if a and b might be equal. +// Returns false if a and b are definitely not equal. +// Requires len(a)>=8. +func longStringQuickEqualityTest(a, b string) bool { + if len(a) != len(b) { + return false + } + x, y := stringPtr(a), stringPtr(b) + // Check first 8 bytes. + if *(*[8]byte)(x) != *(*[8]byte)(y) { + return false + } + // Check last 8 bytes. + x = unsafe.Pointer(uintptr(x) + uintptr(len(a)) - 8) + y = unsafe.Pointer(uintptr(y) + uintptr(len(a)) - 8) + if *(*[8]byte)(x) != *(*[8]byte)(y) { + return false + } + return true +} +func stringPtr(s string) unsafe.Pointer { + type stringStruct struct { + ptr unsafe.Pointer + len int + } + return (*stringStruct)(unsafe.Pointer(&s)).ptr +} + +//go:linkname runtime_mapaccess1_faststr runtime.mapaccess1_faststr +func runtime_mapaccess1_faststr(typ *abi.SwissMapType, m *Map, key string) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen <= 0 { + elem := m.getWithoutKeySmallFastStr(typ, key) + if elem == nil { + return unsafe.Pointer(&zeroVal[0]) + } + return elem + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_faststr runtime.mapaccess2_faststr +func runtime_mapaccess2_faststr(typ *abi.SwissMapType, m *Map, key string) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen <= 0 { + elem := m.getWithoutKeySmallFastStr(typ, key) + if elem == nil { + return unsafe.Pointer(&zeroVal[0]), false + } + return elem, true + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFastStr(typ *abi.SwissMapType, hash uintptr, key string) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + // Key needs update, as the backing storage may differ. + *(*string)(slotKey) = key + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*string)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_faststr runtime.mapassign_faststr +func runtime_mapassign_faststr(typ *abi.SwissMapType, m *Map, key string) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.SwissMapGroupSlots { + elem := m.putSlotSmallFastStr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + // Key needs update, as the backing + // storage may differ. + *(*string)(slotKey) = key + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*string)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_faststr runtime.mapdelete_faststr +func runtime_mapdelete_faststr(typ *abi.SwissMapType, m *Map, key string) { + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/src/internal/runtime/maps/runtime_noswiss.go b/src/internal/runtime/maps/runtime_noswiss.go new file mode 100644 index 00000000..c9342e08 --- /dev/null +++ b/src/internal/runtime/maps/runtime_noswiss.go @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +package maps + +import ( + "internal/abi" + "unsafe" +) + +// For testing, we don't ever need key errors. +func mapKeyError(typ *abi.SwissMapType, p unsafe.Pointer) error { + return nil +} diff --git a/src/internal/runtime/maps/runtime_swiss.go b/src/internal/runtime/maps/runtime_swiss.go new file mode 100644 index 00000000..3f4f970f --- /dev/null +++ b/src/internal/runtime/maps/runtime_swiss.go @@ -0,0 +1,355 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package maps + +import ( + "internal/abi" + "internal/asan" + "internal/msan" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +// Functions below pushed from runtime. + +//go:linkname mapKeyError +func mapKeyError(typ *abi.SwissMapType, p unsafe.Pointer) error + +// Pushed from runtime in order to use runtime.plainError +// +//go:linkname errNilAssign +var errNilAssign error + +// Pull from runtime. It is important that is this the exact same copy as the +// runtime because runtime.mapaccess1_fat compares the returned pointer with +// &runtime.zeroVal[0]. +// TODO: move zeroVal to internal/abi? +// +//go:linkname zeroVal runtime.zeroVal +var zeroVal [abi.ZeroValSize]byte + +// mapaccess1 returns a pointer to h[key]. Never returns nil, instead +// it will return a reference to the zero object for the elem type if +// the key is not in the map. +// NOTE: The returned pointer may keep the whole map live, so don't +// hold onto it for very long. +// +//go:linkname runtime_mapaccess1 runtime.mapaccess1 +func runtime_mapaccess1(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + race.ReadObjectPC(typ.Key, key, callerpc, pc) + } + if msan.Enabled && m != nil { + msan.Read(key, typ.Key.Size_) + } + if asan.Enabled && m != nil { + asan.Read(key, typ.Key.Size_) + } + + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen <= 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + if !ok { + return unsafe.Pointer(&zeroVal[0]) + } + return elem + } + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2 runtime.mapaccess2 +func runtime_mapaccess2(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + race.ReadObjectPC(typ.Key, key, callerpc, pc) + } + if msan.Enabled && m != nil { + msan.Read(key, typ.Key.Size_) + } + if asan.Enabled && m != nil { + asan.Read(key, typ.Key.Size_) + } + + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + if !ok { + return unsafe.Pointer(&zeroVal[0]), false + } + return elem, true + } + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +//go:linkname runtime_mapassign runtime.mapassign +func runtime_mapassign(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + race.ReadObjectPC(typ.Key, key, callerpc, pc) + } + if msan.Enabled { + msan.Read(key, typ.Key.Size_) + } + if asan.Enabled { + asan.Read(key, typ.Key.Size_) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.SwissMapGroupSlots { + elem := m.putSlotSmall(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + + var i uintptr + + // If we found a deleted slot along the way, we + // can replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } else { + // Otherwise, use the empty slot. + i = match.first() + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2(hash))) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + + // No empty slots in this group. Check for a deleted + // slot, which we'll use if we don't find a match later + // in the probe sequence. + // + // We only need to remember a single deleted slot. + if firstDeletedGroup.data == nil { + // Since we already checked for empty slots + // above, matches here must be deleted slots. + match = g.ctrls().matchEmptyOrDeleted() + if match != 0 { + firstDeletedGroup = g + firstDeletedSlot = match.first() + } + } + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} diff --git a/src/internal/runtime/maps/table.go b/src/internal/runtime/maps/table.go new file mode 100644 index 00000000..cc39c24a --- /dev/null +++ b/src/internal/runtime/maps/table.go @@ -0,0 +1,1154 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package maps implements Go's builtin map type. +package maps + +import ( + "internal/abi" + "internal/goarch" + "unsafe" +) + +// Maximum size of a table before it is split at the directory level. +// +// TODO: Completely made up value. This should be tuned for performance vs grow +// latency. +// TODO: This should likely be based on byte size, as copying costs will +// dominate grow latency for large objects. +const maxTableCapacity = 1024 + +// Ensure the max capacity fits in uint16, used for capacity and growthLeft +// below. +var _ = uint16(maxTableCapacity) + +// table is a Swiss table hash table structure. +// +// Each table is a complete hash table implementation. +// +// Map uses one or more tables to store entries. Extendible hashing (hash +// prefix) is used to select the table to use for a specific key. Using +// multiple tables enables incremental growth by growing only one table at a +// time. +type table struct { + // The number of filled slots (i.e. the number of elements in the table). + used uint16 + + // The total number of slots (always 2^N). Equal to + // `(groups.lengthMask+1)*abi.SwissMapGroupSlots`. + capacity uint16 + + // The number of slots we can still fill without needing to rehash. + // + // We rehash when used + tombstones > loadFactor*capacity, including + // tombstones so the table doesn't overfill with tombstones. This field + // counts down remaining empty slots before the next rehash. + growthLeft uint16 + + // The number of bits used by directory lookups above this table. Note + // that this may be less then globalDepth, if the directory has grown + // but this table has not yet been split. + localDepth uint8 + + // Index of this table in the Map directory. This is the index of the + // _first_ location in the directory. The table may occur in multiple + // sequential indicies. + // + // index is -1 if the table is stale (no longer installed in the + // directory). + index int + + // groups is an array of slot groups. Each group holds abi.SwissMapGroupSlots + // key/elem slots and their control bytes. A table has a fixed size + // groups array. The table is replaced (in rehash) when more space is + // required. + // + // TODO(prattmic): keys and elements are interleaved to maximize + // locality, but it comes at the expense of wasted space for some types + // (consider uint8 key, uint64 element). Consider placing all keys + // together in these cases to save space. + groups groupsReference +} + +func newTable(typ *abi.SwissMapType, capacity uint64, index int, localDepth uint8) *table { + if capacity < abi.SwissMapGroupSlots { + capacity = abi.SwissMapGroupSlots + } + + t := &table{ + index: index, + localDepth: localDepth, + } + + if capacity > maxTableCapacity { + panic("initial table capacity too large") + } + + // N.B. group count must be a power of two for probeSeq to visit every + // group. + capacity, overflow := alignUpPow2(capacity) + if overflow { + panic("rounded-up capacity overflows uint64") + } + + t.reset(typ, uint16(capacity)) + + return t +} + +// reset resets the table with new, empty groups with the specified new total +// capacity. +func (t *table) reset(typ *abi.SwissMapType, capacity uint16) { + groupCount := uint64(capacity) / abi.SwissMapGroupSlots + t.groups = newGroups(typ, groupCount) + t.capacity = capacity + t.resetGrowthLeft() + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + g.ctrls().setEmpty() + } +} + +// Preconditions: table must be empty. +func (t *table) resetGrowthLeft() { + var growthLeft uint16 + if t.capacity == 0 { + // No real reason to support zero capacity table, since an + // empty Map simply won't have a table. + panic("table must have positive capacity") + } else if t.capacity <= abi.SwissMapGroupSlots { + // If the map fits in a single group then we're able to fill all of + // the slots except 1 (an empty slot is needed to terminate find + // operations). + // + // TODO(go.dev/issue/54766): With a special case in probing for + // single-group tables, we could fill all slots. + growthLeft = t.capacity - 1 + } else { + if t.capacity*maxAvgGroupLoad < t.capacity { + // TODO(prattmic): Do something cleaner. + panic("overflow") + } + growthLeft = (t.capacity * maxAvgGroupLoad) / abi.SwissMapGroupSlots + } + t.growthLeft = growthLeft +} + +func (t *table) Used() uint64 { + return uint64(t.used) +} + +// Get performs a lookup of the key that key points to. It returns a pointer to +// the element, or false if the key doesn't exist. +func (t *table) Get(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) (unsafe.Pointer, bool) { + // TODO(prattmic): We could avoid hashing in a variety of special + // cases. + // + // - One entry maps could just directly compare the single entry + // without hashing. + // - String keys could do quick checks of a few bytes before hashing. + hash := typ.Hasher(key, m.seed) + _, elem, ok := t.getWithKey(typ, hash, key) + return elem, ok +} + +// getWithKey performs a lookup of key, returning a pointer to the version of +// the key in the map in addition to the element. +// +// This is relevant when multiple different key values compare equal (e.g., +// +0.0 and -0.0). When a grow occurs during iteration, iteration perform a +// lookup of keys from the old group in the new group in order to correctly +// expose updated elements. For NeedsKeyUpdate keys, iteration also must return +// the new key value, not the old key value. +// hash must be the hash of the key. +func (t *table) getWithKey(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + // To find the location of a key in the table, we compute hash(key). From + // h1(hash(key)) and the capacity, we construct a probeSeq that visits + // every group of slots in some interesting order. See [probeSeq]. + // + // We walk through these indices. At each index, we select the entire + // group starting with that index and extract potential candidates: + // occupied slots with a control byte equal to h2(hash(key)). The key + // at candidate slot i is compared with key; if key == g.slot(i).key + // we are done and return the slot; if there is an empty slot in the + // group, we stop and return an error; otherwise we continue to the + // next probe index. Tombstones (ctrlDeleted) effectively behave like + // full slots that never match the value we're looking for. + // + // The h2 bits ensure when we compare a key we are likely to have + // actually found the object. That is, the chance is low that keys + // compare false. Thus, when we search for an object, we are unlikely + // to call Equal many times. This likelihood can be analyzed as follows + // (assuming that h2 is a random enough hash function). + // + // Let's assume that there are k "wrong" objects that must be examined + // in a probe sequence. For example, when doing a find on an object + // that is in the table, k is the number of objects between the start + // of the probe sequence and the final found object (not including the + // final found object). The expected number of objects with an h2 match + // is then k/128. Measurements and analysis indicate that even at high + // load factors, k is less than 32, meaning that the number of false + // positive comparisons we must perform is less than 1/8 per find. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotKey, slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return nil, nil, false + } + } +} + +func (t *table) getWithoutKey(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return nil, false + } + } +} + +// PutSlot returns a pointer to the element slot where an inserted element +// should be written, and ok if it returned a valid slot. +// +// PutSlot returns ok false if the table was split and the Map needs to find +// the new table. +// +// hash must be the hash of key. +func (t *table) PutSlot(typ *abi.SwissMapType, m *Map, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot we + // find, which we'll use to insert the new entry if necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + t.checkInvariants(typ, m) + return slotElem, true + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2(hash))) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + return slotElem, true + } + + t.rehash(typ, m) + return nil, false + } +} + +// uncheckedPutSlot inserts an entry known not to be in the table. +// This is used for grow/split where we are making a new table from +// entries in an existing table. +// +// Decrements growthLeft and increments used. +// +// Requires that the entry does not exist in the table, and that the table has +// room for another element without rehashing. +// +// Requires that there are no deleted entries in the table. +// +// For indirect keys and/or elements, the key and elem pointers can be +// put directly into the map, they do not need to be copied. This +// requires the caller to ensure that the referenced memory never +// changes (by sourcing those pointers from another indirect key/elem +// map). +func (t *table) uncheckedPutSlot(typ *abi.SwissMapType, hash uintptr, key, elem unsafe.Pointer) { + if t.growthLeft == 0 { + panic("invariant failed: growthLeft is unexpectedly 0") + } + + // Given key and its hash hash(key), to insert it, we construct a + // probeSeq, and use it to find the first group with an unoccupied (empty + // or deleted) slot. We place the key/value into the first such slot in + // the group and mark it as full with key's H2. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchEmptyOrDeleted() + if match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + *(*unsafe.Pointer)(slotKey) = key + } else { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + *(*unsafe.Pointer)(slotElem) = elem + } else { + typedmemmove(typ.Elem, slotElem, elem) + } + + t.growthLeft-- + t.used++ + g.ctrls().set(i, ctrl(h2(hash))) + return + } + } +} + +func (t *table) Delete(typ *abi.SwissMapType, m *Map, hash uintptr, key unsafe.Pointer) { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + origSlotKey := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + + if typ.Key.Equal(key, slotKey) { + t.used-- + m.used-- + + if typ.IndirectKey() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(origSlotKey) = nil + } else if typ.Key.Pointers() { + // Only bothing clear the key if there + // are pointers in it. + typedmemclr(typ.Key, slotKey) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(slotElem) = nil + } else { + // Unlike keys, always clear the elem (even if + // it contains no pointers), as compound + // assignment operations depend on cleared + // deleted values. See + // https://go.dev/issue/25936. + typedmemclr(typ.Elem, slotElem) + } + + // Only a full group can appear in the middle + // of a probe sequence (a group with at least + // one empty slot terminates probing). Once a + // group becomes full, it stays full until + // rehashing/resizing. So if the group isn't + // full now, we can simply remove the element. + // Otherwise, we create a tombstone to mark the + // slot as deleted. + if g.ctrls().matchEmpty() != 0 { + g.ctrls().set(i, ctrlEmpty) + t.growthLeft++ + } else { + g.ctrls().set(i, ctrlDeleted) + } + + t.checkInvariants(typ, m) + return + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return + } + } +} + +// tombstones returns the number of deleted (tombstone) entries in the table. A +// tombstone is a slot that has been deleted but is still considered occupied +// so as not to violate the probing invariant. +func (t *table) tombstones() uint16 { + return (t.capacity*maxAvgGroupLoad)/abi.SwissMapGroupSlots - t.used - t.growthLeft +} + +// Clear deletes all entries from the map resulting in an empty map. +func (t *table) Clear(typ *abi.SwissMapType) { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + typedmemclr(typ.Group, g.data) + g.ctrls().setEmpty() + } + + t.used = 0 + t.resetGrowthLeft() +} + +type Iter struct { + key unsafe.Pointer // Must be in first position. Write nil to indicate iteration end (see cmd/compile/internal/walk/range.go). + elem unsafe.Pointer // Must be in second position (see cmd/compile/internal/walk/range.go). + typ *abi.SwissMapType + m *Map + + // Randomize iteration order by starting iteration at a random slot + // offset. The offset into the directory uses a separate offset, as it + // must adjust when the directory grows. + entryOffset uint64 + dirOffset uint64 + + // Snapshot of Map.clearSeq at iteration initialization time. Used to + // detect clear during iteration. + clearSeq uint64 + + // Value of Map.globalDepth during the last call to Next. Used to + // detect directory grow during iteration. + globalDepth uint8 + + // dirIdx is the current directory index, prior to adjustment by + // dirOffset. + dirIdx int + + // tab is the table at dirIdx during the previous call to Next. + tab *table + + // group is the group at entryIdx during the previous call to Next. + group groupReference + + // entryIdx is the current entry index, prior to adjustment by entryOffset. + // The lower 3 bits of the index are the slot index, and the upper bits + // are the group index. + entryIdx uint64 +} + +// Init initializes Iter for iteration. +func (it *Iter) Init(typ *abi.SwissMapType, m *Map) { + it.typ = typ + + if m == nil || m.used == 0 { + return + } + + dirIdx := 0 + var groupSmall groupReference + if m.dirLen <= 0 { + // Use dirIdx == -1 as sentinel for small maps. + dirIdx = -1 + groupSmall.data = m.dirPtr + } + + it.m = m + it.entryOffset = rand() + it.dirOffset = rand() + it.globalDepth = m.globalDepth + it.dirIdx = dirIdx + it.group = groupSmall + it.clearSeq = m.clearSeq +} + +func (it *Iter) Initialized() bool { + return it.typ != nil +} + +// Map returns the map this iterator is iterating over. +func (it *Iter) Map() *Map { + return it.m +} + +// Key returns a pointer to the current key. nil indicates end of iteration. +// +// Must not be called prior to Next. +func (it *Iter) Key() unsafe.Pointer { + return it.key +} + +// Key returns a pointer to the current element. nil indicates end of +// iteration. +// +// Must not be called prior to Next. +func (it *Iter) Elem() unsafe.Pointer { + return it.elem +} + +func (it *Iter) nextDirIdx() { + // Skip other entries in the directory that refer to the same + // logical table. There are two cases of this: + // + // Consider this directory: + // + // - 0: *t1 + // - 1: *t1 + // - 2: *t2a + // - 3: *t2b + // + // At some point, the directory grew to accommodate a split of + // t2. t1 did not split, so entries 0 and 1 both point to t1. + // t2 did split, so the two halves were installed in entries 2 + // and 3. + // + // If dirIdx is 0 and it.tab is t1, then we should skip past + // entry 1 to avoid repeating t1. + // + // If dirIdx is 2 and it.tab is t2 (pre-split), then we should + // skip past entry 3 because our pre-split t2 already covers + // all keys from t2a and t2b (except for new insertions, which + // iteration need not return). + // + // We can achieve both of these by using to difference between + // the directory and table depth to compute how many entries + // the table covers. + entries := 1 << (it.m.globalDepth - it.tab.localDepth) + it.dirIdx += entries + it.tab = nil + it.group = groupReference{} + it.entryIdx = 0 +} + +// Return the appropriate key/elem for key at slotIdx index within it.group, if +// any. +func (it *Iter) grownKeyElem(key unsafe.Pointer, slotIdx uintptr) (unsafe.Pointer, unsafe.Pointer, bool) { + newKey, newElem, ok := it.m.getWithKey(it.typ, key) + if !ok { + // Key has likely been deleted, and + // should be skipped. + // + // One exception is keys that don't + // compare equal to themselves (e.g., + // NaN). These keys cannot be looked + // up, so getWithKey will fail even if + // the key exists. + // + // However, we are in luck because such + // keys cannot be updated and they + // cannot be deleted except with clear. + // Thus if no clear has occurred, the + // key/elem must still exist exactly as + // in the old groups, so we can return + // them from there. + // + // TODO(prattmic): Consider checking + // clearSeq early. If a clear occurred, + // Next could always return + // immediately, as iteration doesn't + // need to return anything added after + // clear. + if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) { + elem := it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + return key, elem, true + } + + // This entry doesn't exist anymore. + return nil, nil, false + } + + return newKey, newElem, true +} + +// Next proceeds to the next element in iteration, which can be accessed via +// the Key and Elem methods. +// +// The table can be mutated during iteration, though there is no guarantee that +// the mutations will be visible to the iteration. +// +// Init must be called prior to Next. +func (it *Iter) Next() { + if it.m == nil { + // Map was empty at Iter.Init. + it.key = nil + it.elem = nil + return + } + + if it.m.writing != 0 { + fatal("concurrent map iteration and map write") + return + } + + if it.dirIdx < 0 { + // Map was small at Init. + for ; it.entryIdx < abi.SwissMapGroupSlots; it.entryIdx++ { + k := uintptr(it.entryIdx+it.entryOffset) % abi.SwissMapGroupSlots + + if (it.group.ctrls().get(k) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted. + continue + } + + key := it.group.key(it.typ, k) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // As below, if we have grown to a full map since Init, + // we continue to use the old group to decide the keys + // to return, but must look them up again in the new + // tables. + grown := it.m.dirLen > 0 + var elem unsafe.Pointer + if grown { + var ok bool + newKey, newElem, ok := it.m.getWithKey(it.typ, key) + if !ok { + // See comment below. + if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) { + elem = it.group.elem(it.typ, k) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } else { + continue + } + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, k) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + it.entryIdx++ + it.key = key + it.elem = elem + return + } + it.key = nil + it.elem = nil + return + } + + if it.globalDepth != it.m.globalDepth { + // Directory has grown since the last call to Next. Adjust our + // directory index. + // + // Consider: + // + // Before: + // - 0: *t1 + // - 1: *t2 <- dirIdx + // + // After: + // - 0: *t1a (split) + // - 1: *t1b (split) + // - 2: *t2 <- dirIdx + // - 3: *t2 + // + // That is, we want to double the current index when the + // directory size doubles (or quadruple when the directory size + // quadruples, etc). + // + // The actual (randomized) dirIdx is computed below as: + // + // dirIdx := (it.dirIdx + it.dirOffset) % it.m.dirLen + // + // Multiplication is associative across modulo operations, + // A * (B % C) = (A * B) % (A * C), + // provided that A is positive. + // + // Thus we can achieve this by adjusting it.dirIdx, + // it.dirOffset, and it.m.dirLen individually. + orders := it.m.globalDepth - it.globalDepth + it.dirIdx <<= orders + it.dirOffset <<= orders + // it.m.dirLen was already adjusted when the directory grew. + + it.globalDepth = it.m.globalDepth + } + + // Continue iteration until we find a full slot. + for ; it.dirIdx < it.m.dirLen; it.nextDirIdx() { + // Resolve the table. + if it.tab == nil { + dirIdx := int((uint64(it.dirIdx) + it.dirOffset) & uint64(it.m.dirLen-1)) + newTab := it.m.directoryAt(uintptr(dirIdx)) + if newTab.index != dirIdx { + // Normally we skip past all duplicates of the + // same entry in the table (see updates to + // it.dirIdx at the end of the loop below), so + // this case wouldn't occur. + // + // But on the very first call, we have a + // completely randomized dirIdx that may refer + // to a middle of a run of tables in the + // directory. Do a one-time adjustment of the + // offset to ensure we start at first index for + // newTable. + diff := dirIdx - newTab.index + it.dirOffset -= uint64(diff) + dirIdx = newTab.index + } + it.tab = newTab + } + + // N.B. Use it.tab, not newTab. It is important to use the old + // table for key selection if the table has grown. See comment + // on grown below. + + entryMask := uint64(it.tab.capacity) - 1 + if it.entryIdx > entryMask { + // Continue to next table. + continue + } + + // Fast path: skip matching and directly check if entryIdx is a + // full slot. + // + // In the slow path below, we perform an 8-slot match check to + // look for full slots within the group. + // + // However, with a max load factor of 7/8, each slot in a + // mostly full map has a high probability of being full. Thus + // it is cheaper to check a single slot than do a full control + // match. + + entryIdx := (it.entryIdx + it.entryOffset) & entryMask + slotIdx := uintptr(entryIdx & (abi.SwissMapGroupSlots - 1)) + if slotIdx == 0 || it.group.data == nil { + // Only compute the group (a) when we switch + // groups (slotIdx rolls over) and (b) on the + // first iteration in this table (slotIdx may + // not be zero due to entryOffset). + groupIdx := entryIdx >> abi.SwissMapGroupSlotsBits + it.group = it.tab.groups.group(it.typ, groupIdx) + } + + if (it.group.ctrls().get(slotIdx) & ctrlEmpty) == 0 { + // Slot full. + + key := it.group.key(it.typ, slotIdx) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + grown := it.tab.index == -1 + var elem unsafe.Pointer + if grown { + newKey, newElem, ok := it.grownKeyElem(key, slotIdx) + if !ok { + // This entry doesn't exist + // anymore. Continue to the + // next one. + goto next + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + it.entryIdx++ + it.key = key + it.elem = elem + return + } + + next: + it.entryIdx++ + + // Slow path: use a match on the control word to jump ahead to + // the next full slot. + // + // This is highly effective for maps with particularly low load + // (e.g., map allocated with large hint but few insertions). + // + // For maps with medium load (e.g., 3-4 empty slots per group) + // it also tends to work pretty well. Since slots within a + // group are filled in order, then if there have been no + // deletions, a match will allow skipping past all empty slots + // at once. + // + // Note: it is tempting to cache the group match result in the + // iterator to use across Next calls. However because entries + // may be deleted between calls later calls would still need to + // double-check the control value. + + var groupMatch bitset + for it.entryIdx <= entryMask { + entryIdx := (it.entryIdx + it.entryOffset) & entryMask + slotIdx := uintptr(entryIdx & (abi.SwissMapGroupSlots - 1)) + + if slotIdx == 0 || it.group.data == nil { + // Only compute the group (a) when we switch + // groups (slotIdx rolls over) and (b) on the + // first iteration in this table (slotIdx may + // not be zero due to entryOffset). + groupIdx := entryIdx >> abi.SwissMapGroupSlotsBits + it.group = it.tab.groups.group(it.typ, groupIdx) + } + + if groupMatch == 0 { + groupMatch = it.group.ctrls().matchFull() + + if slotIdx != 0 { + // Starting in the middle of the group. + // Ignore earlier groups. + groupMatch = groupMatch.removeBelow(slotIdx) + } + + // Skip over groups that are composed of only empty or + // deleted slots. + if groupMatch == 0 { + // Jump past remaining slots in this + // group. + it.entryIdx += abi.SwissMapGroupSlots - uint64(slotIdx) + continue + } + + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + if it.entryIdx > entryMask { + // Past the end of this table's iteration. + continue + } + entryIdx += uint64(i - slotIdx) + slotIdx = i + } + + key := it.group.key(it.typ, slotIdx) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // If the table has changed since the last + // call, then it has grown or split. In this + // case, further mutations (changes to + // key->elem or deletions) will not be visible + // in our snapshot table. Instead we must + // consult the new table by doing a full + // lookup. + // + // We still use our old table to decide which + // keys to lookup in order to avoid returning + // the same key twice. + grown := it.tab.index == -1 + var elem unsafe.Pointer + if grown { + newKey, newElem, ok := it.grownKeyElem(key, slotIdx) + if !ok { + // This entry doesn't exist anymore. + // Continue to the next one. + groupMatch = groupMatch.removeFirst() + if groupMatch == 0 { + // No more entries in this + // group. Continue to next + // group. + it.entryIdx += abi.SwissMapGroupSlots - uint64(slotIdx) + continue + } + + // Next full slot. + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + continue + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + // Jump ahead to the next full slot or next group. + groupMatch = groupMatch.removeFirst() + if groupMatch == 0 { + // No more entries in + // this group. Continue + // to next group. + it.entryIdx += abi.SwissMapGroupSlots - uint64(slotIdx) + } else { + // Next full slot. + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + } + + it.key = key + it.elem = elem + return + } + + // Continue to next table. + } + + it.key = nil + it.elem = nil + return +} + +// Replaces the table with one larger table or two split tables to fit more +// entries. Since the table is replaced, t is now stale and should not be +// modified. +func (t *table) rehash(typ *abi.SwissMapType, m *Map) { + // TODO(prattmic): SwissTables typically perform a "rehash in place" + // operation which recovers capacity consumed by tombstones without growing + // the table by reordering slots as necessary to maintain the probe + // invariant while eliminating all tombstones. + // + // However, it is unclear how to make rehash in place work with + // iteration. Since iteration simply walks through all slots in order + // (with random start offset), reordering the slots would break + // iteration. + // + // As an alternative, we could do a "resize" to new groups allocation + // of the same size. This would eliminate the tombstones, but using a + // new allocation, so the existing grow support in iteration would + // continue to work. + + newCapacity := 2 * t.capacity + if newCapacity <= maxTableCapacity { + t.grow(typ, m, newCapacity) + return + } + + t.split(typ, m) +} + +// Bitmask for the last selection bit at this depth. +func localDepthMask(localDepth uint8) uintptr { + if goarch.PtrSize == 4 { + return uintptr(1) << (32 - localDepth) + } + return uintptr(1) << (64 - localDepth) +} + +// split the table into two, installing the new tables in the map directory. +func (t *table) split(typ *abi.SwissMapType, m *Map) { + localDepth := t.localDepth + localDepth++ + + // TODO: is this the best capacity? + left := newTable(typ, maxTableCapacity, -1, localDepth) + right := newTable(typ, maxTableCapacity, -1, localDepth) + + // Split in half at the localDepth bit from the top. + mask := localDepthMask(localDepth) + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.SwissMapGroupSlots; j++ { + if (g.ctrls().get(j) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted + continue + } + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, j) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + var newTable *table + if hash&mask == 0 { + newTable = left + } else { + newTable = right + } + newTable.uncheckedPutSlot(typ, hash, key, elem) + } + } + + m.installTableSplit(t, left, right) + t.index = -1 +} + +// grow the capacity of the table by allocating a new table with a bigger array +// and uncheckedPutting each element of the table into the new table (we know +// that no insertion here will Put an already-present value), and discard the +// old table. +func (t *table) grow(typ *abi.SwissMapType, m *Map, newCapacity uint16) { + newTable := newTable(typ, uint64(newCapacity), t.index, t.localDepth) + + if t.capacity > 0 { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.SwissMapGroupSlots; j++ { + if (g.ctrls().get(j) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted + continue + } + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, j) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + + newTable.uncheckedPutSlot(typ, hash, key, elem) + } + } + } + + newTable.checkInvariants(typ, m) + m.replaceTable(newTable) + t.index = -1 +} + +// probeSeq maintains the state for a probe sequence that iterates through the +// groups in a table. The sequence is a triangular progression of the form +// +// p(i) := (i^2 + i)/2 + hash (mod mask+1) +// +// The sequence effectively outputs the indexes of *groups*. The group +// machinery allows us to check an entire group with minimal branching. +// +// It turns out that this probe sequence visits every group exactly once if +// the number of groups is a power of two, since (i^2+i)/2 is a bijection in +// Z/(2^m). See https://en.wikipedia.org/wiki/Quadratic_probing +type probeSeq struct { + mask uint64 + offset uint64 + index uint64 +} + +func makeProbeSeq(hash uintptr, mask uint64) probeSeq { + return probeSeq{ + mask: mask, + offset: uint64(hash) & mask, + index: 0, + } +} + +func (s probeSeq) next() probeSeq { + s.index++ + s.offset = (s.offset + s.index) & s.mask + return s +} diff --git a/src/internal/runtime/maps/table_debug.go b/src/internal/runtime/maps/table_debug.go new file mode 100644 index 00000000..a754592f --- /dev/null +++ b/src/internal/runtime/maps/table_debug.go @@ -0,0 +1,131 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package maps implements Go's builtin map type. +package maps + +import ( + "internal/abi" + "unsafe" +) + +const debugLog = false + +func (t *table) checkInvariants(typ *abi.SwissMapType, m *Map) { + if !debugLog { + return + } + + // For every non-empty slot, verify we can retrieve the key using Get. + // Count the number of used and deleted slots. + var used uint16 + var deleted uint16 + var empty uint16 + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.SwissMapGroupSlots; j++ { + c := g.ctrls().get(j) + switch { + case c == ctrlDeleted: + deleted++ + case c == ctrlEmpty: + empty++ + default: + used++ + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // Can't lookup keys that don't compare equal + // to themselves (e.g., NaN). + if !typ.Key.Equal(key, key) { + continue + } + + if _, ok := t.Get(typ, m, key); !ok { + hash := typ.Hasher(key, m.seed) + print("invariant failed: slot(", i, "/", j, "): key ") + dump(key, typ.Key.Size_) + print(" not found [hash=", hash, ", h2=", h2(hash), " h1=", h1(hash), "]\n") + t.Print(typ, m) + panic("invariant failed: slot: key not found") + } + } + } + } + + if used != t.used { + print("invariant failed: found ", used, " used slots, but used count is ", t.used, "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched used slot count") + } + + growthLeft := (t.capacity*maxAvgGroupLoad)/abi.SwissMapGroupSlots - t.used - deleted + if growthLeft != t.growthLeft { + print("invariant failed: found ", t.growthLeft, " growthLeft, but expected ", growthLeft, "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched growthLeft") + } + if deleted != t.tombstones() { + print("invariant failed: found ", deleted, " tombstones, but expected ", t.tombstones(), "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched tombstones") + } + + if empty == 0 { + print("invariant failed: found no empty slots (violates probe invariant)\n") + t.Print(typ, m) + panic("invariant failed: found no empty slots (violates probe invariant)") + } +} +func (t *table) Print(typ *abi.SwissMapType, m *Map) { + print(`table{ + index: `, t.index, ` + localDepth: `, t.localDepth, ` + capacity: `, t.capacity, ` + used: `, t.used, ` + growthLeft: `, t.growthLeft, ` + groups: +`) + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + print("\t\tgroup ", i, "\n") + + g := t.groups.group(typ, i) + ctrls := g.ctrls() + for j := uintptr(0); j < abi.SwissMapGroupSlots; j++ { + print("\t\t\tslot ", j, "\n") + + c := ctrls.get(j) + print("\t\t\t\tctrl ", c) + switch c { + case ctrlEmpty: + print(" (empty)\n") + case ctrlDeleted: + print(" (deleted)\n") + default: + print("\n") + } + + print("\t\t\t\tkey ") + dump(g.key(typ, j), typ.Key.Size_) + println("") + print("\t\t\t\telem ") + dump(g.elem(typ, j), typ.Elem.Size_) + println("") + } + } +} + +// TODO(prattmic): not in hex because print doesn't have a way to print in hex +// outside the runtime. +func dump(ptr unsafe.Pointer, size uintptr) { + for size > 0 { + print(*(*byte)(ptr), " ") + ptr = unsafe.Pointer(uintptr(ptr) + 1) + size-- + } +} diff --git a/src/runtime/internal/math/math.go b/src/internal/runtime/math/math.go similarity index 100% rename from src/runtime/internal/math/math.go rename to src/internal/runtime/math/math.go diff --git a/src/runtime/internal/math/math_test.go b/src/internal/runtime/math/math_test.go similarity index 98% rename from src/runtime/internal/math/math_test.go rename to src/internal/runtime/math/math_test.go index 303eb634..8405999d 100644 --- a/src/runtime/internal/math/math_test.go +++ b/src/internal/runtime/math/math_test.go @@ -5,7 +5,7 @@ package math_test import ( - . "runtime/internal/math" + . "internal/runtime/math" "testing" ) diff --git a/src/runtime/internal/sys/consts.go b/src/internal/runtime/sys/consts.go similarity index 91% rename from src/runtime/internal/sys/consts.go rename to src/internal/runtime/sys/consts.go index 98c0f09e..96e630ed 100644 --- a/src/runtime/internal/sys/consts.go +++ b/src/internal/runtime/sys/consts.go @@ -9,10 +9,10 @@ import ( "internal/goos" ) -// AIX requires a larger stack for syscalls. +// AIX and OpenBSD require a larger stack for syscalls. // The race build also needs more stack. See issue 54291. // This arithmetic must match that in cmd/internal/objabi/stack.go:stackGuardMultiplier. -const StackGuardMultiplier = 1 + goos.IsAix + isRace +const StackGuardMultiplier = 1 + goos.IsAix + goos.IsOpenbsd + isRace // DefaultPhysPageSize is the default physical page size. const DefaultPhysPageSize = goarch.DefaultPhysPageSize diff --git a/src/runtime/internal/sys/consts_norace.go b/src/internal/runtime/sys/consts_norace.go similarity index 100% rename from src/runtime/internal/sys/consts_norace.go rename to src/internal/runtime/sys/consts_norace.go diff --git a/src/runtime/internal/sys/consts_race.go b/src/internal/runtime/sys/consts_race.go similarity index 100% rename from src/runtime/internal/sys/consts_race.go rename to src/internal/runtime/sys/consts_race.go diff --git a/src/internal/runtime/sys/dit_arm64.go b/src/internal/runtime/sys/dit_arm64.go new file mode 100644 index 00000000..643fd770 --- /dev/null +++ b/src/internal/runtime/sys/dit_arm64.go @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 + +package sys + +import ( + "internal/cpu" +) + +var DITSupported = cpu.ARM64.HasDIT + +func EnableDIT() bool +func DITEnabled() bool +func DisableDIT() diff --git a/src/internal/runtime/sys/dit_arm64.s b/src/internal/runtime/sys/dit_arm64.s new file mode 100644 index 00000000..c27dfc9a --- /dev/null +++ b/src/internal/runtime/sys/dit_arm64.s @@ -0,0 +1,22 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT ·EnableDIT(SB),$0-1 + MRS DIT, R0 + UBFX $24, R0, $1, R1 + MOVB R1, ret+0(FP) + MSR $1, DIT + RET + +TEXT ·DITEnabled(SB),$0-1 + MRS DIT, R0 + UBFX $24, R0, $1, R1 + MOVB R1, ret+0(FP) + RET + +TEXT ·DisableDIT(SB),$0 + MSR $0, DIT + RET diff --git a/src/internal/runtime/sys/empty.s b/src/internal/runtime/sys/empty.s new file mode 100644 index 00000000..3e62b7da --- /dev/null +++ b/src/internal/runtime/sys/empty.s @@ -0,0 +1 @@ +// Empty assembly file to allow empty function bodies for intrinsics. diff --git a/src/runtime/internal/sys/intrinsics.go b/src/internal/runtime/sys/intrinsics.go similarity index 81% rename from src/runtime/internal/sys/intrinsics.go rename to src/internal/runtime/sys/intrinsics.go index e6a37584..147d5581 100644 --- a/src/runtime/internal/sys/intrinsics.go +++ b/src/internal/runtime/sys/intrinsics.go @@ -206,3 +206,51 @@ func Prefetch(addr uintptr) {} // // ARM64: Produce PRFM instruction with PLDL1STRM option func PrefetchStreamed(addr uintptr) {} + +// GetCallerPC returns the program counter (PC) of its caller's caller. +// GetCallerSP returns the stack pointer (SP) of its caller's caller. +// Both are implemented as intrinsics on every platform. +// +// For example: +// +// func f(arg1, arg2, arg3 int) { +// pc := GetCallerPC() +// sp := GetCallerSP() +// } +// +// These two lines find the PC and SP immediately following +// the call to f (where f will return). +// +// The call to GetCallerPC and GetCallerSP must be done in the +// frame being asked about. +// +// The result of GetCallerSP is correct at the time of the return, +// but it may be invalidated by any subsequent call to a function +// that might relocate the stack in order to grow or shrink it. +// A general rule is that the result of GetCallerSP should be used +// immediately and can only be passed to nosplit functions. + +func GetCallerPC() uintptr + +func GetCallerSP() uintptr + +// GetClosurePtr returns the pointer to the current closure. +// GetClosurePtr can only be used in an assignment statement +// at the entry of a function. Moreover, go:nosplit directive +// must be specified at the declaration of caller function, +// so that the function prolog does not clobber the closure register. +// for example: +// +// //go:nosplit +// func f(arg1, arg2, arg3 int) { +// dx := GetClosurePtr() +// } +// +// The compiler rewrites calls to this function into instructions that fetch the +// pointer from a well-known register (DX on x86 architecture, etc.) directly. +// +// WARNING: PGO-based devirtualization cannot detect that caller of +// GetClosurePtr requires closure context, and thus must maintain a list of +// these functions, which is in +// cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. +func GetClosurePtr() uintptr diff --git a/src/runtime/internal/sys/intrinsics_test.go b/src/internal/runtime/sys/intrinsics_test.go similarity index 97% rename from src/runtime/internal/sys/intrinsics_test.go rename to src/internal/runtime/sys/intrinsics_test.go index 67998850..d466f3e5 100644 --- a/src/runtime/internal/sys/intrinsics_test.go +++ b/src/internal/runtime/sys/intrinsics_test.go @@ -5,7 +5,7 @@ package sys_test import ( - "runtime/internal/sys" + "internal/runtime/sys" "testing" ) diff --git a/src/runtime/internal/sys/nih.go b/src/internal/runtime/sys/nih.go similarity index 95% rename from src/runtime/internal/sys/nih.go rename to src/internal/runtime/sys/nih.go index 17eab673..a9cbc48e 100644 --- a/src/runtime/internal/sys/nih.go +++ b/src/internal/runtime/sys/nih.go @@ -14,7 +14,7 @@ type nih struct{} // Other types can embed NotInHeap to make it not-in-heap. Specifically, pointers // to these types must always fail the `runtime.inheap` check. The type may be used // for global variables, or for objects in unmanaged memory (e.g., allocated with -// `sysAlloc`, `persistentalloc`, r`fixalloc`, or from a manually-managed span). +// `sysAlloc`, `persistentalloc`, `fixalloc`, or from a manually-managed span). // // Specifically: // diff --git a/src/internal/runtime/sys/no_dit.go b/src/internal/runtime/sys/no_dit.go new file mode 100644 index 00000000..0589d0ca --- /dev/null +++ b/src/internal/runtime/sys/no_dit.go @@ -0,0 +1,13 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !arm64 + +package sys + +var DITSupported = false + +func EnableDIT() bool { return false } +func DITEnabled() bool { return false } +func DisableDIT() {} diff --git a/src/runtime/internal/sys/sys.go b/src/internal/runtime/sys/sys.go similarity index 100% rename from src/runtime/internal/sys/sys.go rename to src/internal/runtime/sys/sys.go diff --git a/src/internal/stringslite/strings.go b/src/internal/stringslite/strings.go index 4114b861..3a09e08c 100644 --- a/src/internal/stringslite/strings.go +++ b/src/internal/stringslite/strings.go @@ -14,7 +14,7 @@ import ( ) func HasPrefix(s, prefix string) bool { - return len(s) >= len(prefix) && s[0:len(prefix)] == prefix + return len(s) >= len(prefix) && s[:len(prefix)] == prefix } func HasSuffix(s, suffix string) bool { diff --git a/src/internal/sync/export_test.go b/src/internal/sync/export_test.go new file mode 100644 index 00000000..c6449bf3 --- /dev/null +++ b/src/internal/sync/export_test.go @@ -0,0 +1,38 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import ( + "internal/abi" + "unsafe" +) + +// NewBadHashTrieMap creates a new HashTrieMap for the provided key and value +// but with an intentionally bad hash function. +func NewBadHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + var m HashTrieMap[K, V] + m.init() + m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr { + return 0 + } + return &m +} + +// NewTruncHashTrieMap creates a new HashTrieMap for the provided key and value +// but with an intentionally bad hash function. +func NewTruncHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + var m HashTrieMap[K, V] + var mx map[string]int + mapType := abi.TypeOf(mx).MapType() + hasher := mapType.Hasher + m.keyHash = func(p unsafe.Pointer, n uintptr) uintptr { + return hasher(p, n) & ((uintptr(1) << 4) - 1) + } + return &m +} diff --git a/src/internal/sync/hashtriemap.go b/src/internal/sync/hashtriemap.go new file mode 100644 index 00000000..6f5e0b43 --- /dev/null +++ b/src/internal/sync/hashtriemap.go @@ -0,0 +1,724 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import ( + "internal/abi" + "internal/goarch" + "sync/atomic" + "unsafe" +) + +// HashTrieMap is an implementation of a concurrent hash-trie. The implementation +// is designed around frequent loads, but offers decent performance for stores +// and deletes as well, especially if the map is larger. Its primary use-case is +// the unique package, but can be used elsewhere as well. +// +// The zero HashTrieMap is empty and ready to use. +// It must not be copied after first use. +type HashTrieMap[K comparable, V any] struct { + inited atomic.Uint32 + initMu Mutex + root atomic.Pointer[indirect[K, V]] + keyHash hashFunc + valEqual equalFunc + seed uintptr +} + +func (ht *HashTrieMap[K, V]) init() { + if ht.inited.Load() == 0 { + ht.initSlow() + } +} + +//go:noinline +func (ht *HashTrieMap[K, V]) initSlow() { + ht.initMu.Lock() + defer ht.initMu.Unlock() + + if ht.inited.Load() != 0 { + // Someone got to it while we were waiting. + return + } + + // Set up root node, derive the hash function for the key, and the + // equal function for the value, if any. + var m map[K]V + mapType := abi.TypeOf(m).MapType() + ht.root.Store(newIndirectNode[K, V](nil)) + ht.keyHash = mapType.Hasher + ht.valEqual = mapType.Elem.Equal + ht.seed = uintptr(runtime_rand()) + + ht.inited.Store(1) +} + +type hashFunc func(unsafe.Pointer, uintptr) uintptr +type equalFunc func(unsafe.Pointer, unsafe.Pointer) bool + +// Load returns the value stored in the map for a key, or nil if no +// value is present. +// The ok result indicates whether value was found in the map. +func (ht *HashTrieMap[K, V]) Load(key K) (value V, ok bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + i := ht.root.Load() + hashShift := 8 * goarch.PtrSize + for hashShift != 0 { + hashShift -= nChildrenLog2 + + n := i.children[(hash>>hashShift)&nChildrenMask].Load() + if n == nil { + return *new(V), false + } + if n.isEntry { + return n.entry().lookup(key) + } + i = n.indirect() + } + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + var i *indirect[K, V] + var hashShift uint + var slot *atomic.Pointer[node[K, V]] + var n *node[K, V] + for { + // Find the key or a candidate location for insertion. + i = ht.root.Load() + hashShift = 8 * goarch.PtrSize + haveInsertPoint := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil { + // We found a nil slot which is a candidate for insertion. + haveInsertPoint = true + break + } + if n.isEntry { + // We found an existing entry, which is as far as we can go. + // If it stays this way, we'll have to replace it with an + // indirect node. + if v, ok := n.entry().lookup(key); ok { + return v, true + } + haveInsertPoint = true + break + } + i = n.indirect() + } + if !haveInsertPoint { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + // What we saw is still true, so we can continue with the insert. + break + } + // We have to start over. + i.mu.Unlock() + } + // N.B. This lock is held from when we broke out of the outer loop above. + // We specifically break this out so that we can use defer here safely. + // One option is to break this out into a new function instead, but + // there's so much local iteration state used below that this turns out + // to be cleaner. + defer i.mu.Unlock() + + var oldEntry *entry[K, V] + if n != nil { + oldEntry = n.entry() + if v, ok := oldEntry.lookup(key); ok { + // Easy case: by loading again, it turns out exactly what we wanted is here! + return v, true + } + } + newEntry := newEntryNode(key, value) + if oldEntry == nil { + // Easy case: create a new entry and store it. + slot.Store(&newEntry.node) + } else { + // We possibly need to expand the entry already there into one or more new nodes. + // + // Publish the node last, which will make both oldEntry and newEntry visible. We + // don't want readers to be able to observe that oldEntry isn't in the tree. + slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i)) + } + return value, false +} + +// expand takes oldEntry and newEntry whose hashes conflict from bit 64 down to hashShift and +// produces a subtree of indirect nodes to hold the two new entries. +func (ht *HashTrieMap[K, V]) expand(oldEntry, newEntry *entry[K, V], newHash uintptr, hashShift uint, parent *indirect[K, V]) *node[K, V] { + // Check for a hash collision. + oldHash := ht.keyHash(unsafe.Pointer(&oldEntry.key), ht.seed) + if oldHash == newHash { + // Store the old entry in the new entry's overflow list, then store + // the new entry. + newEntry.overflow.Store(oldEntry) + return &newEntry.node + } + // We have to add an indirect node. Worse still, we may need to add more than one. + newIndirect := newIndirectNode(parent) + top := newIndirect + for { + if hashShift == 0 { + panic("internal/sync.HashTrieMap: ran out of hash bits while inserting") + } + hashShift -= nChildrenLog2 // hashShift is for the level parent is at. We need to go deeper. + oi := (oldHash >> hashShift) & nChildrenMask + ni := (newHash >> hashShift) & nChildrenMask + if oi != ni { + newIndirect.children[oi].Store(&oldEntry.node) + newIndirect.children[ni].Store(&newEntry.node) + break + } + nextIndirect := newIndirectNode(newIndirect) + newIndirect.children[oi].Store(&nextIndirect.node) + newIndirect = nextIndirect + } + return &top.node +} + +// Store sets the value for a key. +func (ht *HashTrieMap[K, V]) Store(key K, old V) { + _, _ = ht.Swap(key, old) +} + +// Swap swaps the value for a key and returns the previous value if any. +// The loaded result reports whether the key was present. +func (ht *HashTrieMap[K, V]) Swap(key K, new V) (previous V, loaded bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + var i *indirect[K, V] + var hashShift uint + var slot *atomic.Pointer[node[K, V]] + var n *node[K, V] + for { + // Find the key or a candidate location for insertion. + i = ht.root.Load() + hashShift = 8 * goarch.PtrSize + haveInsertPoint := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil || n.isEntry { + // We found a nil slot which is a candidate for insertion, + // or an existing entry that we'll replace. + haveInsertPoint = true + break + } + i = n.indirect() + } + if !haveInsertPoint { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + // What we saw is still true, so we can continue with the insert. + break + } + // We have to start over. + i.mu.Unlock() + } + // N.B. This lock is held from when we broke out of the outer loop above. + // We specifically break this out so that we can use defer here safely. + // One option is to break this out into a new function instead, but + // there's so much local iteration state used below that this turns out + // to be cleaner. + defer i.mu.Unlock() + + var zero V + var oldEntry *entry[K, V] + if n != nil { + // Swap if the keys compare. + oldEntry = n.entry() + newEntry, old, swapped := oldEntry.swap(key, new) + if swapped { + slot.Store(&newEntry.node) + return old, true + } + } + // The keys didn't compare, so we're doing an insertion. + newEntry := newEntryNode(key, new) + if oldEntry == nil { + // Easy case: create a new entry and store it. + slot.Store(&newEntry.node) + } else { + // We possibly need to expand the entry already there into one or more new nodes. + // + // Publish the node last, which will make both oldEntry and newEntry visible. We + // don't want readers to be able to observe that oldEntry isn't in the tree. + slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i)) + } + return zero, false +} + +// CompareAndSwap swaps the old and new values for key +// if the value stored in the map is equal to old. +// The value type must be of a comparable type, otherwise CompareAndSwap will panic. +func (ht *HashTrieMap[K, V]) CompareAndSwap(key K, old, new V) (swapped bool) { + ht.init() + if ht.valEqual == nil { + panic("called CompareAndSwap when value is not of comparable type") + } + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key and compare with it. n != nil if we found the node. + i, _, slot, n := ht.find(key, hash, ht.valEqual, old) + if i != nil { + defer i.mu.Unlock() + } + if n == nil { + return false + } + + // Try to swap the entry. + e, swapped := n.entry().compareAndSwap(key, old, new, ht.valEqual) + if !swapped { + // Nothing was actually swapped, which means the node is no longer there. + return false + } + // Store the entry back because it changed. + slot.Store(&e.node) + return true +} + +// LoadAndDelete deletes the value for a key, returning the previous value if any. +// The loaded result reports whether the key was present. +func (ht *HashTrieMap[K, V]) LoadAndDelete(key K) (value V, loaded bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key and compare with it. n != nil if we found the node. + i, hashShift, slot, n := ht.find(key, hash, nil, *new(V)) + if n == nil { + if i != nil { + i.mu.Unlock() + } + return *new(V), false + } + + // Try to delete the entry. + v, e, loaded := n.entry().loadAndDelete(key) + if !loaded { + // Nothing was actually deleted, which means the node is no longer there. + i.mu.Unlock() + return *new(V), false + } + if e != nil { + // We didn't actually delete the whole entry, just one entry in the chain. + // Nothing else to do, since the parent is definitely not empty. + slot.Store(&e.node) + i.mu.Unlock() + return v, true + } + // Delete the entry. + slot.Store(nil) + + // Check if the node is now empty (and isn't the root), and delete it if able. + for i.parent != nil && i.empty() { + if hashShift == 8*goarch.PtrSize { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + hashShift += nChildrenLog2 + + // Delete the current node in the parent. + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return v, true +} + +// Delete deletes the value for a key. +func (ht *HashTrieMap[K, V]) Delete(key K) { + _, _ = ht.LoadAndDelete(key) +} + +// CompareAndDelete deletes the entry for key if its value is equal to old. +// The value type must be comparable, otherwise this CompareAndDelete will panic. +// +// If there is no current value for key in the map, CompareAndDelete returns false +// (even if the old value is the nil interface value). +func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { + ht.init() + if ht.valEqual == nil { + panic("called CompareAndDelete when value is not of comparable type") + } + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key. n != nil if we found the node. + i, hashShift, slot, n := ht.find(key, hash, nil, *new(V)) + if n == nil { + if i != nil { + i.mu.Unlock() + } + return false + } + + // Try to delete the entry. + e, deleted := n.entry().compareAndDelete(key, old, ht.valEqual) + if !deleted { + // Nothing was actually deleted, which means the node is no longer there. + i.mu.Unlock() + return false + } + if e != nil { + // We didn't actually delete the whole entry, just one entry in the chain. + // Nothing else to do, since the parent is definitely not empty. + slot.Store(&e.node) + i.mu.Unlock() + return true + } + // Delete the entry. + slot.Store(nil) + + // Check if the node is now empty (and isn't the root), and delete it if able. + for i.parent != nil && i.empty() { + if hashShift == 8*goarch.PtrSize { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + hashShift += nChildrenLog2 + + // Delete the current node in the parent. + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return true +} + +// find searches the tree for a node that contains key (hash must be the hash of key). +// If valEqual != nil, then it will also enforce that the values are equal as well. +// +// Returns a non-nil node, which will always be an entry, if found. +// +// If i != nil then i.mu is locked, and it is the caller's responsibility to unlock it. +func (ht *HashTrieMap[K, V]) find(key K, hash uintptr, valEqual equalFunc, value V) (i *indirect[K, V], hashShift uint, slot *atomic.Pointer[node[K, V]], n *node[K, V]) { + for { + // Find the key or return if it's not there. + i = ht.root.Load() + hashShift = 8 * goarch.PtrSize + found := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil { + // Nothing to compare with. Give up. + i = nil + return + } + if n.isEntry { + // We found an entry. Check if it matches. + if _, ok := n.entry().lookupWithValue(key, value, valEqual); !ok { + // No match, comparison failed. + i = nil + n = nil + return + } + // We've got a match. Prepare to perform an operation on the key. + found = true + break + } + i = n.indirect() + } + if !found { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if !i.dead.Load() && (n == nil || n.isEntry) { + // Either we've got a valid node or the node is now nil under the lock. + // In either case, we're done here. + return + } + // We have to start over. + i.mu.Unlock() + } +} + +// All returns an iterator over each key and value present in the map. +// +// The iterator does not necessarily correspond to any consistent snapshot of the +// HashTrieMap's contents: no key will be visited more than once, but if the value +// for any key is stored or deleted concurrently (including by yield), the iterator +// may reflect any mapping for that key from any point during iteration. The iterator +// does not block other methods on the receiver; even yield itself may call any +// method on the HashTrieMap. +func (ht *HashTrieMap[K, V]) All() func(yield func(K, V) bool) { + ht.init() + return func(yield func(key K, value V) bool) { + ht.iter(ht.root.Load(), yield) + } +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// This exists for compatibility with sync.Map; All should be preferred. +// It provides the same guarantees as sync.Map, and All. +func (ht *HashTrieMap[K, V]) Range(yield func(K, V) bool) { + ht.init() + ht.iter(ht.root.Load(), yield) +} + +func (ht *HashTrieMap[K, V]) iter(i *indirect[K, V], yield func(key K, value V) bool) bool { + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + if !n.isEntry { + if !ht.iter(n.indirect(), yield) { + return false + } + continue + } + e := n.entry() + for e != nil { + if !yield(e.key, e.value) { + return false + } + e = e.overflow.Load() + } + } + return true +} + +// Clear deletes all the entries, resulting in an empty HashTrieMap. +func (ht *HashTrieMap[K, V]) Clear() { + ht.init() + + // It's sufficient to just drop the root on the floor, but the root + // must always be non-nil. + ht.root.Store(newIndirectNode[K, V](nil)) +} + +const ( + // 16 children. This seems to be the sweet spot for + // load performance: any smaller and we lose out on + // 50% or more in CPU performance. Any larger and the + // returns are minuscule (~1% improvement for 32 children). + nChildrenLog2 = 4 + nChildren = 1 << nChildrenLog2 + nChildrenMask = nChildren - 1 +) + +// indirect is an internal node in the hash-trie. +type indirect[K comparable, V any] struct { + node[K, V] + dead atomic.Bool + mu Mutex // Protects mutation to children and any children that are entry nodes. + parent *indirect[K, V] + children [nChildren]atomic.Pointer[node[K, V]] +} + +func newIndirectNode[K comparable, V any](parent *indirect[K, V]) *indirect[K, V] { + return &indirect[K, V]{node: node[K, V]{isEntry: false}, parent: parent} +} + +func (i *indirect[K, V]) empty() bool { + nc := 0 + for j := range i.children { + if i.children[j].Load() != nil { + nc++ + } + } + return nc == 0 +} + +// entry is a leaf node in the hash-trie. +type entry[K comparable, V any] struct { + node[K, V] + overflow atomic.Pointer[entry[K, V]] // Overflow for hash collisions. + key K + value V +} + +func newEntryNode[K comparable, V any](key K, value V) *entry[K, V] { + return &entry[K, V]{ + node: node[K, V]{isEntry: true}, + key: key, + value: value, + } +} + +func (e *entry[K, V]) lookup(key K) (V, bool) { + for e != nil { + if e.key == key { + return e.value, true + } + e = e.overflow.Load() + } + return *new(V), false +} + +func (e *entry[K, V]) lookupWithValue(key K, value V, valEqual equalFunc) (V, bool) { + for e != nil { + if e.key == key && (valEqual == nil || valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value)))) { + return e.value, true + } + e = e.overflow.Load() + } + return *new(V), false +} + +// swap replaces an entry in the overflow chain if keys compare equal. Returns the new entry chain, +// the old value, and whether or not anything was swapped. +// +// swap must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) swap(key K, new V) (*entry[K, V], V, bool) { + if head.key == key { + // Return the new head of the list. + e := newEntryNode(key, new) + if chain := head.overflow.Load(); chain != nil { + e.overflow.Store(chain) + } + return e, head.value, true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key { + eNew := newEntryNode(key, new) + eNew.overflow.Store(e.overflow.Load()) + i.Store(eNew) + return head, e.value, true + } + i = &e.overflow + e = e.overflow.Load() + } + var zero V + return head, zero, false +} + +// compareAndSwap replaces an entry in the overflow chain if both the key and value compare +// equal. Returns the new entry chain and whether or not anything was swapped. +// +// compareAndSwap must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) compareAndSwap(key K, old, new V, valEqual equalFunc) (*entry[K, V], bool) { + if head.key == key && valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&old))) { + // Return the new head of the list. + e := newEntryNode(key, new) + if chain := head.overflow.Load(); chain != nil { + e.overflow.Store(chain) + } + return e, true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key && valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&old))) { + eNew := newEntryNode(key, new) + eNew.overflow.Store(e.overflow.Load()) + i.Store(eNew) + return head, true + } + i = &e.overflow + e = e.overflow.Load() + } + return head, false +} + +// loadAndDelete deletes an entry in the overflow chain by key. Returns the value for the key, the new +// entry chain and whether or not anything was loaded (and deleted). +// +// loadAndDelete must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) loadAndDelete(key K) (V, *entry[K, V], bool) { + if head.key == key { + // Drop the head of the list. + return head.value, head.overflow.Load(), true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key { + i.Store(e.overflow.Load()) + return e.value, head, true + } + i = &e.overflow + e = e.overflow.Load() + } + return *new(V), head, false +} + +// compareAndDelete deletes an entry in the overflow chain if both the key and value compare +// equal. Returns the new entry chain and whether or not anything was deleted. +// +// compareAndDelete must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) compareAndDelete(key K, value V, valEqual equalFunc) (*entry[K, V], bool) { + if head.key == key && valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&value))) { + // Drop the head of the list. + return head.overflow.Load(), true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key && valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value))) { + i.Store(e.overflow.Load()) + return head, true + } + i = &e.overflow + e = e.overflow.Load() + } + return head, false +} + +// node is the header for a node. It's polymorphic and +// is actually either an entry or an indirect. +type node[K comparable, V any] struct { + isEntry bool +} + +func (n *node[K, V]) entry() *entry[K, V] { + if !n.isEntry { + panic("called entry on non-entry node") + } + return (*entry[K, V])(unsafe.Pointer(n)) +} + +func (n *node[K, V]) indirect() *indirect[K, V] { + if n.isEntry { + panic("called indirect on entry node") + } + return (*indirect[K, V])(unsafe.Pointer(n)) +} + +// Pull in runtime.rand so that we don't need to take a dependency +// on math/rand/v2. +// +//go:linkname runtime_rand runtime.rand +func runtime_rand() uint64 diff --git a/src/internal/concurrent/hashtriemap_bench_test.go b/src/internal/sync/hashtriemap_bench_test.go similarity index 89% rename from src/internal/concurrent/hashtriemap_bench_test.go rename to src/internal/sync/hashtriemap_bench_test.go index 32a263d5..fc10d822 100644 --- a/src/internal/concurrent/hashtriemap_bench_test.go +++ b/src/internal/sync/hashtriemap_bench_test.go @@ -2,9 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package concurrent +package sync_test -import "testing" +import ( + isync "internal/sync" + "testing" +) func BenchmarkHashTrieMapLoadSmall(b *testing.B) { benchmarkHashTrieMapLoad(b, testDataSmall[:]) @@ -20,7 +23,7 @@ func BenchmarkHashTrieMapLoadLarge(b *testing.B) { func benchmarkHashTrieMapLoad(b *testing.B, data []string) { b.ReportAllocs() - m := NewHashTrieMap[string, int]() + var m isync.HashTrieMap[string, int] for i := range data { m.LoadOrStore(data[i], i) } @@ -47,7 +50,7 @@ func BenchmarkHashTrieMapLoadOrStoreLarge(b *testing.B) { func benchmarkHashTrieMapLoadOrStore(b *testing.B, data []string) { b.ReportAllocs() - m := NewHashTrieMap[string, int]() + var m isync.HashTrieMap[string, int] b.RunParallel(func(pb *testing.PB) { i := 0 diff --git a/src/internal/sync/hashtriemap_test.go b/src/internal/sync/hashtriemap_test.go new file mode 100644 index 00000000..d9219f84 --- /dev/null +++ b/src/internal/sync/hashtriemap_test.go @@ -0,0 +1,982 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync_test + +import ( + "fmt" + isync "internal/sync" + "math" + "runtime" + "strconv" + "sync" + "testing" + "weak" +) + +func TestHashTrieMap(t *testing.T) { + testHashTrieMap(t, func() *isync.HashTrieMap[string, int] { + var m isync.HashTrieMap[string, int] + return &m + }) +} + +func TestHashTrieMapBadHash(t *testing.T) { + testHashTrieMap(t, func() *isync.HashTrieMap[string, int] { + return isync.NewBadHashTrieMap[string, int]() + }) +} + +func TestHashTrieMapTruncHash(t *testing.T) { + testHashTrieMap(t, func() *isync.HashTrieMap[string, int] { + // Stub out the good hash function with a different terrible one + // (truncated hash). Everything should still work as expected. + // This is useful to test independently to catch issues with + // near collisions, where only the last few bits of the hash differ. + return isync.NewTruncHashTrieMap[string, int]() + }) +} + +func testHashTrieMap(t *testing.T, newMap func() *isync.HashTrieMap[string, int]) { + t.Run("LoadEmpty", func(t *testing.T) { + m := newMap() + + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("LoadOrStore", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + }) + t.Run("All", func(t *testing.T) { + m := newMap() + + testAll(t, m, testDataMap(testData[:]), func(_ string, _ int) bool { + return true + }) + }) + t.Run("Clear", func(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + m.Clear() + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("Concurrent", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for _, s := range testData { + // Try a couple things to interfere with the clear. + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + m.CompareAndSwap(s, i, i+1) // May succeed or fail; we don't care. + } + }(i) + } + + // Concurrently clear the map. + runtime.Gosched() + m.Clear() + + // Wait for workers to finish. + wg.Wait() + + // It should all be empty now. + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + }) + t.Run("CompareAndDelete", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for range 3 { + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) + expectNotDeleted(t, s, i)(m.CompareAndDelete(s, i)) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectNotDeleted(t, testData[15], math.MaxInt)(m.CompareAndDelete(testData[15], math.MaxInt)) + expectDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) + expectNotDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) + for i, s := range testData { + if i == 15 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectNotDeleted(t, testData[i], math.MaxInt)(m.CompareAndDelete(testData[i], math.MaxInt)) + expectDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) + expectNotDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Iterate", func(t *testing.T) { + m := newMap() + + testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { + expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) + return true + }) + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectDeleted(t, key, id)(m.CompareAndDelete(key, id)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + m.CompareAndDelete(s, i) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("CompareAndSwap", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for j := range 3 { + for i, s := range testData { + expectPresent(t, s, i+j)(m.Load(s)) + expectNotSwapped(t, s, math.MaxInt, i+j+1)(m.CompareAndSwap(s, math.MaxInt, i+j+1)) + expectSwapped(t, s, i, i+j+1)(m.CompareAndSwap(s, i+j, i+j+1)) + expectNotSwapped(t, s, i+j, i+j+1)(m.CompareAndSwap(s, i+j, i+j+1)) + expectPresent(t, s, i+j+1)(m.Load(s)) + } + } + for i, s := range testData { + expectPresent(t, s, i+3)(m.Load(s)) + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectNotSwapped(t, testData[15], math.MaxInt, 16)(m.CompareAndSwap(testData[15], math.MaxInt, 16)) + expectSwapped(t, testData[15], 15, 16)(m.CompareAndSwap(testData[15], 15, 16)) + expectNotSwapped(t, testData[15], 15, 16)(m.CompareAndSwap(testData[15], 15, 16)) + for i, s := range testData { + if i == 15 { + expectPresent(t, s, 16)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectNotSwapped(t, testData[i], math.MaxInt, i+1)(m.CompareAndSwap(testData[i], math.MaxInt, i+1)) + expectSwapped(t, testData[i], i, i+1)(m.CompareAndSwap(testData[i], i, i+1)) + expectNotSwapped(t, testData[i], i, i+1)(m.CompareAndSwap(testData[i], i, i+1)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectPresent(t, s, i+1)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectSwapped(t, key, id, id+1)(m.CompareAndSwap(key, id, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+1)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentUnsharedKeysWithDelete", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectSwapped(t, key, id, id+1)(m.CompareAndSwap(key, id, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + expectDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectNotSwapped(t, key, id+1, id+2)(m.CompareAndSwap(key, id+1, id+2)) + expectNotDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + expectNotSwapped(t, s, math.MaxInt, i+1)(m.CompareAndSwap(s, math.MaxInt, i+1)) + m.CompareAndSwap(s, i, i+1) + expectPresent(t, s, i+1)(m.Load(s)) + } + for i, s := range testData { + expectPresent(t, s, i+1)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("Swap", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + for j := range 3 { + for i, s := range testData { + expectPresent(t, s, i+j)(m.Load(s)) + expectLoadedFromSwap(t, s, i+j, i+j+1)(m.Swap(s, i+j+1)) + expectPresent(t, s, i+j+1)(m.Load(s)) + } + } + for i, s := range testData { + expectLoadedFromSwap(t, s, i+3, i+3)(m.Swap(s, i+3)) + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + expectLoadedFromSwap(t, testData[15], 15, 16)(m.Swap(testData[15], 16)) + for i, s := range testData { + if i == 15 { + expectPresent(t, s, 16)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + for _, i := range []int{1, 105, 6, 85} { + expectLoadedFromSwap(t, testData[i], i, i+1)(m.Swap(testData[i], i+1)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectPresent(t, s, i+1)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectNotLoadedFromSwap(t, key, id)(m.Swap(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id)(m.Swap(key, id)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id+1)(m.Swap(key, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+1)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentUnsharedKeysWithDelete", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectNotLoadedFromSwap(t, key, id)(m.Swap(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id)(m.Swap(key, id)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id+1)(m.Swap(key, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + expectDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectNotLoadedFromSwap(t, key, id+2)(m.Swap(key, id+2)) + expectPresent(t, key, id+2)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+2)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + m.Swap(s, i+1) + expectPresent(t, s, i+1)(m.Load(s)) + } + for i, s := range testData { + expectPresent(t, s, i+1)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("LoadAndDelete", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for range 3 { + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromDelete(t, s, i)(m.LoadAndDelete(s)) + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromDelete(t, s, 0)(m.LoadAndDelete(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectPresent(t, testData[15], 15)(m.Load(testData[15])) + expectLoadedFromDelete(t, testData[15], 15)(m.LoadAndDelete(testData[15])) + expectMissing(t, testData[15], 0)(m.Load(testData[15])) + expectNotLoadedFromDelete(t, testData[15], 0)(m.LoadAndDelete(testData[15])) + for i, s := range testData { + if i == 15 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectPresent(t, testData[i], i)(m.Load(testData[i])) + expectLoadedFromDelete(t, testData[i], i)(m.LoadAndDelete(testData[i])) + expectMissing(t, testData[i], 0)(m.Load(testData[i])) + expectNotLoadedFromDelete(t, testData[i], 0)(m.LoadAndDelete(testData[i])) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Iterate", func(t *testing.T) { + m := newMap() + + testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { + expectLoadedFromDelete(t, s, i)(m.LoadAndDelete(s)) + return true + }) + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromDelete(t, key, id)(m.LoadAndDelete(key)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for _, s := range testData { + m.LoadAndDelete(s) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) +} + +func testAll[K, V comparable](t *testing.T, m *isync.HashTrieMap[K, V], testData map[K]V, yield func(K, V) bool) { + for k, v := range testData { + expectStored(t, k, v)(m.LoadOrStore(k, v)) + } + visited := make(map[K]int) + m.All()(func(key K, got V) bool { + want, ok := testData[key] + if !ok { + t.Errorf("unexpected key %v in map", key) + return false + } + if got != want { + t.Errorf("expected key %v to have value %v, got %v", key, want, got) + return false + } + visited[key]++ + return yield(key, got) + }) + for key, n := range visited { + if n > 1 { + t.Errorf("visited key %v more than once", key) + } + } +} + +func expectPresent[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { + t.Helper() + return func(got V, ok bool) { + t.Helper() + + if !ok { + t.Errorf("expected key %v to be present in map", key) + } + if ok && got != want { + t.Errorf("expected key %v to have value %v, got %v", key, want, got) + } + } +} + +func expectMissing[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { + t.Helper() + if want != *new(V) { + // This is awkward, but the want argument is necessary to smooth over type inference. + // Just make sure the want argument always looks the same. + panic("expectMissing must always have a zero value variable") + } + return func(got V, ok bool) { + t.Helper() + + if ok { + t.Errorf("expected key %v to be missing from map, got value %v", key, got) + } + if !ok && got != want { + t.Errorf("expected missing key %v to be paired with the zero value; got %v", key, got) + } + } +} + +func expectLoaded[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to have been loaded, not stored", key) + } + if got != want { + t.Errorf("expected key %v to have value %v, got %v", key, want, got) + } + } +} + +func expectStored[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected inserted key %v to have been stored, not loaded", key) + } + if got != want { + t.Errorf("expected inserted key %v to have value %v, got %v", key, want, got) + } + } +} + +func expectDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { + t.Helper() + return func(deleted bool) { + t.Helper() + + if !deleted { + t.Errorf("expected key %v with value %v to be in map and deleted", key, old) + } + } +} + +func expectNotDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { + t.Helper() + return func(deleted bool) { + t.Helper() + + if deleted { + t.Errorf("expected key %v with value %v to not be in map and thus not deleted", key, old) + } + } +} + +func expectSwapped[K, V comparable](t *testing.T, key K, old, new V) func(swapped bool) { + t.Helper() + return func(swapped bool) { + t.Helper() + + if !swapped { + t.Errorf("expected key %v with value %v to be in map and swapped for %v", key, old, new) + } + } +} + +func expectNotSwapped[K, V comparable](t *testing.T, key K, old, new V) func(swapped bool) { + t.Helper() + return func(swapped bool) { + t.Helper() + + if swapped { + t.Errorf("expected key %v with value %v to not be in map or not swapped for %v", key, old, new) + } + } +} + +func expectLoadedFromSwap[K, V comparable](t *testing.T, key K, want, new V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to be in map and for %v to have been swapped for %v", key, want, new) + } else if want != got { + t.Errorf("key %v had its value %v swapped for %v, but expected it to have value %v", key, got, new, want) + } + } +} + +func expectNotLoadedFromSwap[K, V comparable](t *testing.T, key K, new V) func(old V, loaded bool) { + t.Helper() + return func(old V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected key %v to not be in map, but found value %v for it", key, old) + } + } +} + +func expectLoadedFromDelete[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to be in map to be deleted", key) + } else if want != got { + t.Errorf("key %v was deleted with value %v, but expected it to have value %v", key, got, want) + } + } +} + +func expectNotLoadedFromDelete[K, V comparable](t *testing.T, key K, _ V) func(old V, loaded bool) { + t.Helper() + return func(old V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected key %v to not be in map, but found value %v for it", key, old) + } + } +} + +func testDataMap(data []string) map[string]int { + m := make(map[string]int) + for i, s := range data { + m[s] = i + } + return m +} + +var ( + testDataSmall [8]string + testData [128]string + testDataLarge [128 << 10]string +) + +func init() { + for i := range testDataSmall { + testDataSmall[i] = fmt.Sprintf("%b", i) + } + for i := range testData { + testData[i] = fmt.Sprintf("%b", i) + } + for i := range testDataLarge { + testDataLarge[i] = fmt.Sprintf("%b", i) + } +} + +// TestConcurrentCache tests HashTrieMap in a scenario where it is used as +// the basis of a memory-efficient concurrent cache. We're specifically +// looking to make sure that CompareAndSwap and CompareAndDelete are +// atomic with respect to one another. When competing for the same +// key-value pair, they must not both succeed. +// +// This test is a regression test for issue #70970. +func TestConcurrentCache(t *testing.T) { + type dummy [32]byte + + var m isync.HashTrieMap[int, weak.Pointer[dummy]] + + type cleanupArg struct { + key int + value weak.Pointer[dummy] + } + cleanup := func(arg cleanupArg) { + m.CompareAndDelete(arg.key, arg.value) + } + get := func(m *isync.HashTrieMap[int, weak.Pointer[dummy]], key int) *dummy { + nv := new(dummy) + nw := weak.Make(nv) + for { + w, loaded := m.LoadOrStore(key, nw) + if !loaded { + runtime.AddCleanup(nv, cleanup, cleanupArg{key, nw}) + return nv + } + if v := w.Value(); v != nil { + return v + } + + // Weak pointer was reclaimed, try to replace it with nw. + if m.CompareAndSwap(key, w, nw) { + runtime.AddCleanup(nv, cleanup, cleanupArg{key, nw}) + return nv + } + } + } + + const N = 100_000 + const P = 5_000 + + var wg sync.WaitGroup + wg.Add(N) + for i := range N { + go func() { + defer wg.Done() + a := get(&m, i%P) + b := get(&m, i%P) + if a != b { + t.Errorf("consecutive cache reads returned different values: a != b (%p vs %p)\n", a, b) + } + }() + } + wg.Wait() +} diff --git a/src/internal/sync/mutex.go b/src/internal/sync/mutex.go new file mode 100644 index 00000000..c0c526a7 --- /dev/null +++ b/src/internal/sync/mutex.go @@ -0,0 +1,234 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sync provides basic synchronization primitives such as mutual +// exclusion locks to internal packages (including ones that depend on sync). +// +// Tests are defined in package [sync]. +package sync + +import ( + "internal/race" + "sync/atomic" + "unsafe" +) + +// A Mutex is a mutual exclusion lock. +// +// See package [sync.Mutex] documentation. +type Mutex struct { + state int32 + sema uint32 +} + +const ( + mutexLocked = 1 << iota // mutex is locked + mutexWoken + mutexStarving + mutexWaiterShift = iota + + // Mutex fairness. + // + // Mutex can be in 2 modes of operations: normal and starvation. + // In normal mode waiters are queued in FIFO order, but a woken up waiter + // does not own the mutex and competes with new arriving goroutines over + // the ownership. New arriving goroutines have an advantage -- they are + // already running on CPU and there can be lots of them, so a woken up + // waiter has good chances of losing. In such case it is queued at front + // of the wait queue. If a waiter fails to acquire the mutex for more than 1ms, + // it switches mutex to the starvation mode. + // + // In starvation mode ownership of the mutex is directly handed off from + // the unlocking goroutine to the waiter at the front of the queue. + // New arriving goroutines don't try to acquire the mutex even if it appears + // to be unlocked, and don't try to spin. Instead they queue themselves at + // the tail of the wait queue. + // + // If a waiter receives ownership of the mutex and sees that either + // (1) it is the last waiter in the queue, or (2) it waited for less than 1 ms, + // it switches mutex back to normal operation mode. + // + // Normal mode has considerably better performance as a goroutine can acquire + // a mutex several times in a row even if there are blocked waiters. + // Starvation mode is important to prevent pathological cases of tail latency. + starvationThresholdNs = 1e6 +) + +// Lock locks m. +// +// See package [sync.Mutex] documentation. +func (m *Mutex) Lock() { + // Fast path: grab unlocked mutex. + if atomic.CompareAndSwapInt32(&m.state, 0, mutexLocked) { + if race.Enabled { + race.Acquire(unsafe.Pointer(m)) + } + return + } + // Slow path (outlined so that the fast path can be inlined) + m.lockSlow() +} + +// TryLock tries to lock m and reports whether it succeeded. +// +// See package [sync.Mutex] documentation. +func (m *Mutex) TryLock() bool { + old := m.state + if old&(mutexLocked|mutexStarving) != 0 { + return false + } + + // There may be a goroutine waiting for the mutex, but we are + // running now and can try to grab the mutex before that + // goroutine wakes up. + if !atomic.CompareAndSwapInt32(&m.state, old, old|mutexLocked) { + return false + } + + if race.Enabled { + race.Acquire(unsafe.Pointer(m)) + } + return true +} + +func (m *Mutex) lockSlow() { + var waitStartTime int64 + starving := false + awoke := false + iter := 0 + old := m.state + for { + // Don't spin in starvation mode, ownership is handed off to waiters + // so we won't be able to acquire the mutex anyway. + if old&(mutexLocked|mutexStarving) == mutexLocked && runtime_canSpin(iter) { + // Active spinning makes sense. + // Try to set mutexWoken flag to inform Unlock + // to not wake other blocked goroutines. + if !awoke && old&mutexWoken == 0 && old>>mutexWaiterShift != 0 && + atomic.CompareAndSwapInt32(&m.state, old, old|mutexWoken) { + awoke = true + } + runtime_doSpin() + iter++ + old = m.state + continue + } + new := old + // Don't try to acquire starving mutex, new arriving goroutines must queue. + if old&mutexStarving == 0 { + new |= mutexLocked + } + if old&(mutexLocked|mutexStarving) != 0 { + new += 1 << mutexWaiterShift + } + // The current goroutine switches mutex to starvation mode. + // But if the mutex is currently unlocked, don't do the switch. + // Unlock expects that starving mutex has waiters, which will not + // be true in this case. + if starving && old&mutexLocked != 0 { + new |= mutexStarving + } + if awoke { + // The goroutine has been woken from sleep, + // so we need to reset the flag in either case. + if new&mutexWoken == 0 { + throw("sync: inconsistent mutex state") + } + new &^= mutexWoken + } + if atomic.CompareAndSwapInt32(&m.state, old, new) { + if old&(mutexLocked|mutexStarving) == 0 { + break // locked the mutex with CAS + } + // If we were already waiting before, queue at the front of the queue. + queueLifo := waitStartTime != 0 + if waitStartTime == 0 { + waitStartTime = runtime_nanotime() + } + runtime_SemacquireMutex(&m.sema, queueLifo, 2) + starving = starving || runtime_nanotime()-waitStartTime > starvationThresholdNs + old = m.state + if old&mutexStarving != 0 { + // If this goroutine was woken and mutex is in starvation mode, + // ownership was handed off to us but mutex is in somewhat + // inconsistent state: mutexLocked is not set and we are still + // accounted as waiter. Fix that. + if old&(mutexLocked|mutexWoken) != 0 || old>>mutexWaiterShift == 0 { + throw("sync: inconsistent mutex state") + } + delta := int32(mutexLocked - 1<>mutexWaiterShift == 1 { + // Exit starvation mode. + // Critical to do it here and consider wait time. + // Starvation mode is so inefficient, that two goroutines + // can go lock-step infinitely once they switch mutex + // to starvation mode. + delta -= mutexStarving + } + atomic.AddInt32(&m.state, delta) + break + } + awoke = true + iter = 0 + } else { + old = m.state + } + } + + if race.Enabled { + race.Acquire(unsafe.Pointer(m)) + } +} + +// Unlock unlocks m. +// +// See package [sync.Mutex] documentation. +func (m *Mutex) Unlock() { + if race.Enabled { + _ = m.state + race.Release(unsafe.Pointer(m)) + } + + // Fast path: drop lock bit. + new := atomic.AddInt32(&m.state, -mutexLocked) + if new != 0 { + // Outlined slow path to allow inlining the fast path. + // To hide unlockSlow during tracing we skip one extra frame when tracing GoUnblock. + m.unlockSlow(new) + } +} + +func (m *Mutex) unlockSlow(new int32) { + if (new+mutexLocked)&mutexLocked == 0 { + fatal("sync: unlock of unlocked mutex") + } + if new&mutexStarving == 0 { + old := new + for { + // If there are no waiters or a goroutine has already + // been woken or grabbed the lock, no need to wake anyone. + // In starvation mode ownership is directly handed off from unlocking + // goroutine to the next waiter. We are not part of this chain, + // since we did not observe mutexStarving when we unlocked the mutex above. + // So get off the way. + if old>>mutexWaiterShift == 0 || old&(mutexLocked|mutexWoken|mutexStarving) != 0 { + return + } + // Grab the right to wake someone. + new = (old - 1<= 3 { + break + } + } + }() + want := []time.Time{ + time.Now(), + time.Now().Add(1 * time.Second), + time.Now().Add(2 * time.Second), + } + time.Sleep(5 * time.Second) + synctest.Wait() + if !slices.Equal(got, want) { + t.Errorf("got: %v; want: %v", got, want) + } + }) +} + +func TestIteratorPull(t *testing.T) { + synctest.Run(func() { + seq := func(yield func(time.Time) bool) { + for yield(time.Now()) { + time.Sleep(1 * time.Second) + } + } + var got []time.Time + go func() { + next, stop := iter.Pull(seq) + defer stop() + for len(got) < 3 { + now, _ := next() + got = append(got, now) + } + }() + want := []time.Time{ + time.Now(), + time.Now().Add(1 * time.Second), + time.Now().Add(2 * time.Second), + } + time.Sleep(5 * time.Second) + synctest.Wait() + if !slices.Equal(got, want) { + t.Errorf("got: %v; want: %v", got, want) + } + }) +} + +func TestReflectFuncOf(t *testing.T) { + mkfunc := func(name string, i int) { + reflect.FuncOf([]reflect.Type{ + reflect.StructOf([]reflect.StructField{{ + Name: name + strconv.Itoa(i), + Type: reflect.TypeOf(0), + }}), + }, nil, false) + } + go func() { + for i := 0; i < 100000; i++ { + mkfunc("A", i) + } + }() + synctest.Run(func() { + for i := 0; i < 100000; i++ { + mkfunc("A", i) + } + }) +} + +func TestWaitGroup(t *testing.T) { + synctest.Run(func() { + var wg sync.WaitGroup + wg.Add(1) + const delay = 1 * time.Second + go func() { + time.Sleep(delay) + wg.Done() + }() + start := time.Now() + wg.Wait() + if got := time.Since(start); got != delay { + t.Fatalf("WaitGroup.Wait() took %v, want %v", got, delay) + } + }) +} + +func wantPanic(t *testing.T, want string) { + if e := recover(); e != nil { + if got := fmt.Sprint(e); got != want { + t.Errorf("got panic message %q, want %q", got, want) + } + } else { + t.Errorf("got no panic, want one") + } +} diff --git a/src/internal/syscall/unix/getentropy_netbsd.go b/src/internal/syscall/unix/arandom_netbsd.go similarity index 93% rename from src/internal/syscall/unix/getentropy_netbsd.go rename to src/internal/syscall/unix/arandom_netbsd.go index 02bac1be..23ca8739 100644 --- a/src/internal/syscall/unix/getentropy_netbsd.go +++ b/src/internal/syscall/unix/arandom_netbsd.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build netbsd - package unix import ( @@ -17,7 +15,7 @@ const ( _KERN_ARND = 81 ) -func GetEntropy(p []byte) error { +func Arandom(p []byte) error { mib := [2]uint32{_CTL_KERN, _KERN_ARND} n := uintptr(len(p)) _, _, errno := syscall.Syscall6( diff --git a/src/internal/syscall/unix/arc4random_openbsd.go b/src/internal/syscall/unix/arc4random_openbsd.go new file mode 100644 index 00000000..652e0cb1 --- /dev/null +++ b/src/internal/syscall/unix/arc4random_openbsd.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +//go:linkname syscall_syscall syscall.syscall +func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:cgo_import_dynamic libc_arc4random_buf arc4random_buf "libc.so" + +func libc_arc4random_buf_trampoline() + +func ARC4Random(p []byte) { + syscall_syscall(abi.FuncPCABI0(libc_arc4random_buf_trampoline), + uintptr(unsafe.Pointer(unsafe.SliceData(p))), uintptr(len(p)), 0) +} diff --git a/src/internal/syscall/unix/asm_darwin.s b/src/internal/syscall/unix/asm_darwin.s index 99f28765..b96eb1e8 100644 --- a/src/internal/syscall/unix/asm_darwin.s +++ b/src/internal/syscall/unix/asm_darwin.s @@ -23,3 +23,5 @@ TEXT ·libc_getgrnam_r_trampoline(SB),NOSPLIT,$0-0; JMP libc_getgrnam_r(SB) TEXT ·libc_getgrgid_r_trampoline(SB),NOSPLIT,$0-0; JMP libc_getgrgid_r(SB) TEXT ·libc_sysconf_trampoline(SB),NOSPLIT,$0-0; JMP libc_sysconf(SB) TEXT ·libc_faccessat_trampoline(SB),NOSPLIT,$0-0; JMP libc_faccessat(SB) +TEXT ·libc_readlinkat_trampoline(SB),NOSPLIT,$0-0; JMP libc_readlinkat(SB) +TEXT ·libc_mkdirat_trampoline(SB),NOSPLIT,$0-0; JMP libc_mkdirat(SB) diff --git a/src/internal/syscall/unix/asm_openbsd.s b/src/internal/syscall/unix/asm_openbsd.s index cc54a14c..90f6831e 100644 --- a/src/internal/syscall/unix/asm_openbsd.s +++ b/src/internal/syscall/unix/asm_openbsd.s @@ -8,3 +8,9 @@ TEXT ·libc_faccessat_trampoline(SB),NOSPLIT,$0-0 JMP libc_faccessat(SB) +TEXT ·libc_arc4random_buf_trampoline(SB),NOSPLIT,$0-0 + JMP libc_arc4random_buf(SB) +TEXT ·libc_readlinkat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_readlinkat(SB) +TEXT ·libc_mkdirat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_mkdirat(SB) diff --git a/src/internal/syscall/unix/at.go b/src/internal/syscall/unix/at.go index cfb6e410..27a798e0 100644 --- a/src/internal/syscall/unix/at.go +++ b/src/internal/syscall/unix/at.go @@ -38,3 +38,44 @@ func Openat(dirfd int, path string, flags int, perm uint32) (int, error) { return int(fd), nil } + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall.Syscall6(readlinkatTrap, + uintptr(dirfd), + uintptr(unsafe.Pointer(p0)), + uintptr(p1), + uintptr(len(buf)), + 0, 0) + if errno != 0 { + return 0, errno + } + + return int(n), nil +} + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + _, _, errno := syscall.Syscall6(mkdiratTrap, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + 0, 0, 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_aix.go b/src/internal/syscall/unix/at_aix.go index 3fe3285c..5c2f00ef 100644 --- a/src/internal/syscall/unix/at_aix.go +++ b/src/internal/syscall/unix/at_aix.go @@ -7,8 +7,12 @@ package unix //go:cgo_import_dynamic libc_fstatat fstatat "libc.a/shr_64.o" //go:cgo_import_dynamic libc_openat openat "libc.a/shr_64.o" //go:cgo_import_dynamic libc_unlinkat unlinkat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_readlinkat readlinkat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_mkdirat mkdirat "libc.a/shr_64.o" const ( + AT_EACCESS = 0x1 + AT_FDCWD = -0x02 AT_REMOVEDIR = 0x1 AT_SYMLINK_NOFOLLOW = 0x1 UTIME_OMIT = -0x3 diff --git a/src/internal/syscall/unix/at_darwin.go b/src/internal/syscall/unix/at_darwin.go new file mode 100644 index 00000000..dbcae5a7 --- /dev/null +++ b/src/internal/syscall/unix/at_darwin.go @@ -0,0 +1,60 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +func libc_readlinkat_trampoline() + +//go:cgo_import_dynamic libc_readlinkat readlinkat "/usr/lib/libSystem.B.dylib" + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_readlinkat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p0)), + uintptr(p1), + uintptr(len(buf)), + 0, + 0) + if errno != 0 { + return 0, errno + } + return int(n), nil +} + +func libc_mkdirat_trampoline() + +//go:cgo_import_dynamic libc_mkdirat mkdirat "/usr/lib/libSystem.B.dylib" + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall(abi.FuncPCABI0(libc_mkdirat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode)) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_fstatat.go b/src/internal/syscall/unix/at_fstatat.go index 25de336a..18cd62be 100644 --- a/src/internal/syscall/unix/at_fstatat.go +++ b/src/internal/syscall/unix/at_fstatat.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build dragonfly || (linux && !loong64) || netbsd || (openbsd && mips64) +//go:build dragonfly || (linux && !(loong64 || mips64 || mips64le)) || netbsd || (openbsd && mips64) package unix diff --git a/src/internal/syscall/unix/at_fstatat2.go b/src/internal/syscall/unix/at_fstatat2.go index 8d20e1a8..b09aecbc 100644 --- a/src/internal/syscall/unix/at_fstatat2.go +++ b/src/internal/syscall/unix/at_fstatat2.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build freebsd || (linux && loong64) +//go:build freebsd || (linux && (loong64 || mips64 || mips64le)) package unix diff --git a/src/internal/syscall/unix/at_libc.go b/src/internal/syscall/unix/at_libc.go index f48d3791..faf38be6 100644 --- a/src/internal/syscall/unix/at_libc.go +++ b/src/internal/syscall/unix/at_libc.go @@ -14,11 +14,15 @@ import ( //go:linkname procFstatat libc_fstatat //go:linkname procOpenat libc_openat //go:linkname procUnlinkat libc_unlinkat +//go:linkname procReadlinkat libc_readlinkat +//go:linkname procMkdirat libc_mkdirat var ( procFstatat, procOpenat, - procUnlinkat uintptr + procUnlinkat, + procReadlinkat, + procMkdirat uintptr ) func Unlinkat(dirfd int, path string, flags int) error { @@ -62,3 +66,44 @@ func Fstatat(dirfd int, path string, stat *syscall.Stat_t, flags int) error { return nil } + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall6(uintptr(unsafe.Pointer(&procReadlinkat)), 4, + uintptr(dirfd), + uintptr(unsafe.Pointer(p0)), + uintptr(p1), + uintptr(len(buf)), + 0, 0) + if errno != 0 { + return 0, errno + } + + return int(n), nil +} + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procMkdirat)), 3, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + 0, 0, 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_openbsd.go b/src/internal/syscall/unix/at_openbsd.go new file mode 100644 index 00000000..69463e00 --- /dev/null +++ b/src/internal/syscall/unix/at_openbsd.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build openbsd && !mips64 + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +//go:cgo_import_dynamic libc_readlinkat readlinkat "libc.so" + +func libc_readlinkat_trampoline() + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_readlinkat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p0)), uintptr(p1), uintptr(len(buf)), 0, 0) + if errno != 0 { + return 0, errno + } + return int(n), nil +} + +//go:cgo_import_dynamic libc_mkdirat mkdirat "libc.so" + +func libc_mkdirat_trampoline() + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_mkdirat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p)), 0, 0, 0, 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_solaris.go b/src/internal/syscall/unix/at_solaris.go index ae1c1d64..fa65d9e8 100644 --- a/src/internal/syscall/unix/at_solaris.go +++ b/src/internal/syscall/unix/at_solaris.go @@ -12,12 +12,17 @@ func syscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err // Implemented as rawsysvicall6 in runtime/syscall_solaris.go. func rawSyscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) +//go:cgo_import_dynamic libc_faccessat faccessat "libc.so" //go:cgo_import_dynamic libc_fstatat fstatat "libc.so" //go:cgo_import_dynamic libc_openat openat "libc.so" //go:cgo_import_dynamic libc_unlinkat unlinkat "libc.so" +//go:cgo_import_dynamic libc_readlinkat readlinkat "libc.so" +//go:cgo_import_dynamic libc_mkdirat mkdirat "libc.so" //go:cgo_import_dynamic libc_uname uname "libc.so" const ( + AT_EACCESS = 0x4 + AT_FDCWD = 0xffd19553 AT_REMOVEDIR = 0x1 AT_SYMLINK_NOFOLLOW = 0x1000 diff --git a/src/internal/syscall/unix/at_sysnum_dragonfly.go b/src/internal/syscall/unix/at_sysnum_dragonfly.go index a8164dcc..d0ba12a7 100644 --- a/src/internal/syscall/unix/at_sysnum_dragonfly.go +++ b/src/internal/syscall/unix/at_sysnum_dragonfly.go @@ -7,9 +7,11 @@ package unix import "syscall" const ( - unlinkatTrap uintptr = syscall.SYS_UNLINKAT - openatTrap uintptr = syscall.SYS_OPENAT - fstatatTrap uintptr = syscall.SYS_FSTATAT + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + fstatatTrap uintptr = syscall.SYS_FSTATAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT AT_EACCESS = 0x4 AT_FDCWD = 0xfffafdcd diff --git a/src/internal/syscall/unix/at_sysnum_freebsd.go b/src/internal/syscall/unix/at_sysnum_freebsd.go index f74961d5..0f347224 100644 --- a/src/internal/syscall/unix/at_sysnum_freebsd.go +++ b/src/internal/syscall/unix/at_sysnum_freebsd.go @@ -17,4 +17,6 @@ const ( unlinkatTrap uintptr = syscall.SYS_UNLINKAT openatTrap uintptr = syscall.SYS_OPENAT posixFallocateTrap uintptr = syscall.SYS_POSIX_FALLOCATE + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT ) diff --git a/src/internal/syscall/unix/at_sysnum_linux.go b/src/internal/syscall/unix/at_sysnum_linux.go index 7c3b15c3..2885c7c6 100644 --- a/src/internal/syscall/unix/at_sysnum_linux.go +++ b/src/internal/syscall/unix/at_sysnum_linux.go @@ -6,8 +6,12 @@ package unix import "syscall" -const unlinkatTrap uintptr = syscall.SYS_UNLINKAT -const openatTrap uintptr = syscall.SYS_OPENAT +const ( + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT +) const ( AT_EACCESS = 0x200 diff --git a/src/internal/syscall/unix/at_sysnum_netbsd.go b/src/internal/syscall/unix/at_sysnum_netbsd.go index ffb1d2ea..820b9774 100644 --- a/src/internal/syscall/unix/at_sysnum_netbsd.go +++ b/src/internal/syscall/unix/at_sysnum_netbsd.go @@ -6,9 +6,13 @@ package unix import "syscall" -const unlinkatTrap uintptr = syscall.SYS_UNLINKAT -const openatTrap uintptr = syscall.SYS_OPENAT -const fstatatTrap uintptr = syscall.SYS_FSTATAT +const ( + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + fstatatTrap uintptr = syscall.SYS_FSTATAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT +) const ( AT_EACCESS = 0x100 diff --git a/src/internal/syscall/unix/at_sysnum_openbsd.go b/src/internal/syscall/unix/at_sysnum_openbsd.go index 3b0c0dbd..7672414c 100644 --- a/src/internal/syscall/unix/at_sysnum_openbsd.go +++ b/src/internal/syscall/unix/at_sysnum_openbsd.go @@ -6,9 +6,13 @@ package unix import "syscall" -const unlinkatTrap uintptr = syscall.SYS_UNLINKAT -const openatTrap uintptr = syscall.SYS_OPENAT -const fstatatTrap uintptr = syscall.SYS_FSTATAT +const ( + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + fstatatTrap uintptr = syscall.SYS_FSTATAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT +) const ( AT_EACCESS = 0x1 diff --git a/src/internal/syscall/unix/at_wasip1.go b/src/internal/syscall/unix/at_wasip1.go index 3d47d7eb..cd0cb4b7 100644 --- a/src/internal/syscall/unix/at_wasip1.go +++ b/src/internal/syscall/unix/at_wasip1.go @@ -2,12 +2,112 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build wasip1 + package unix +import ( + "syscall" + "unsafe" +) + +// The values of these constants are not part of the WASI API. const ( // UTIME_OMIT is the sentinel value to indicate that a time value should not // be changed. It is useful for example to indicate for example with UtimesNano // to avoid changing AccessTime or ModifiedTime. // Its value must match syscall/fs_wasip1.go UTIME_OMIT = -0x2 + + AT_REMOVEDIR = 0x200 + AT_SYMLINK_NOFOLLOW = 0x100 ) + +func Unlinkat(dirfd int, path string, flags int) error { + if flags&AT_REMOVEDIR == 0 { + return errnoErr(path_unlink_file( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + )) + } else { + return errnoErr(path_remove_directory( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + )) + } +} + +//go:wasmimport wasi_snapshot_preview1 path_unlink_file +//go:noescape +func path_unlink_file(fd int32, path *byte, pathLen size) syscall.Errno + +//go:wasmimport wasi_snapshot_preview1 path_remove_directory +//go:noescape +func path_remove_directory(fd int32, path *byte, pathLen size) syscall.Errno + +func Openat(dirfd int, path string, flags int, perm uint32) (int, error) { + return syscall.Openat(dirfd, path, flags, perm) +} + +func Fstatat(dirfd int, path string, stat *syscall.Stat_t, flags int) error { + var filestatFlags uint32 + if flags&AT_SYMLINK_NOFOLLOW == 0 { + filestatFlags |= syscall.LOOKUP_SYMLINK_FOLLOW + } + return errnoErr(path_filestat_get( + int32(dirfd), + uint32(filestatFlags), + unsafe.StringData(path), + size(len(path)), + unsafe.Pointer(stat), + )) +} + +//go:wasmimport wasi_snapshot_preview1 path_filestat_get +//go:noescape +func path_filestat_get(fd int32, flags uint32, path *byte, pathLen size, buf unsafe.Pointer) syscall.Errno + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + var nwritten size + errno := path_readlink( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + &buf[0], + size(len(buf)), + &nwritten) + return int(nwritten), errnoErr(errno) + +} + +type ( + size = uint32 +) + +//go:wasmimport wasi_snapshot_preview1 path_readlink +//go:noescape +func path_readlink(fd int32, path *byte, pathLen size, buf *byte, bufLen size, nwritten *size) syscall.Errno + +func Mkdirat(dirfd int, path string, mode uint32) error { + if path == "" { + return syscall.EINVAL + } + return errnoErr(path_create_directory( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + )) +} + +//go:wasmimport wasi_snapshot_preview1 path_create_directory +//go:noescape +func path_create_directory(fd int32, path *byte, pathLen size) syscall.Errno + +func errnoErr(errno syscall.Errno) error { + if errno == 0 { + return nil + } + return errno +} diff --git a/src/internal/syscall/unix/copy_file_range_linux.go b/src/internal/syscall/unix/copy_file_range_unix.go similarity index 95% rename from src/internal/syscall/unix/copy_file_range_linux.go rename to src/internal/syscall/unix/copy_file_range_unix.go index cf0a279a..16a43421 100644 --- a/src/internal/syscall/unix/copy_file_range_linux.go +++ b/src/internal/syscall/unix/copy_file_range_unix.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build freebsd || linux + package unix import ( diff --git a/src/internal/syscall/unix/eaccess.go b/src/internal/syscall/unix/eaccess.go new file mode 100644 index 00000000..3c12314a --- /dev/null +++ b/src/internal/syscall/unix/eaccess.go @@ -0,0 +1,24 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package unix + +import ( + "runtime" + "syscall" +) + +func Eaccess(path string, mode uint32) error { + if runtime.GOOS == "android" { + // syscall.Faccessat for Android implements AT_EACCESS check in + // userspace. Since Android doesn't have setuid programs and + // never runs code with euid!=uid, AT_EACCESS check is not + // really required. Return ENOSYS so the callers can fall back + // to permission bits check. + return syscall.ENOSYS + } + return faccessat(AT_FDCWD, path, mode, AT_EACCESS) +} diff --git a/src/internal/syscall/unix/eaccess_linux.go b/src/internal/syscall/unix/eaccess_linux.go deleted file mode 100644 index 5695a5e4..00000000 --- a/src/internal/syscall/unix/eaccess_linux.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import "syscall" - -func Eaccess(path string, mode uint32) error { - return syscall.Faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/eaccess_other.go b/src/internal/syscall/unix/eaccess_other.go deleted file mode 100644 index 3da3a64f..00000000 --- a/src/internal/syscall/unix/eaccess_other.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build unix && !darwin && !dragonfly && !freebsd && !linux && !openbsd && !netbsd - -package unix - -import "syscall" - -func Eaccess(path string, mode uint32) error { - return syscall.ENOSYS -} diff --git a/src/internal/syscall/unix/eaccess_bsd.go b/src/internal/syscall/unix/faccessat_bsd.go similarity index 85% rename from src/internal/syscall/unix/eaccess_bsd.go rename to src/internal/syscall/unix/faccessat_bsd.go index 7077af17..78fca18e 100644 --- a/src/internal/syscall/unix/eaccess_bsd.go +++ b/src/internal/syscall/unix/faccessat_bsd.go @@ -22,7 +22,3 @@ func faccessat(dirfd int, path string, mode uint32, flags int) error { } return err } - -func Eaccess(path string, mode uint32) error { - return faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/eaccess_darwin.go b/src/internal/syscall/unix/faccessat_darwin.go similarity index 87% rename from src/internal/syscall/unix/eaccess_darwin.go rename to src/internal/syscall/unix/faccessat_darwin.go index 0fa8d17a..ef790aa9 100644 --- a/src/internal/syscall/unix/eaccess_darwin.go +++ b/src/internal/syscall/unix/faccessat_darwin.go @@ -25,7 +25,3 @@ func faccessat(dirfd int, path string, mode uint32, flags int) error { } return nil } - -func Eaccess(path string, mode uint32) error { - return faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/eaccess_openbsd.go b/src/internal/syscall/unix/faccessat_openbsd.go similarity index 89% rename from src/internal/syscall/unix/eaccess_openbsd.go rename to src/internal/syscall/unix/faccessat_openbsd.go index 5e91f11f..9d4ed972 100644 --- a/src/internal/syscall/unix/eaccess_openbsd.go +++ b/src/internal/syscall/unix/faccessat_openbsd.go @@ -30,7 +30,3 @@ func faccessat(dirfd int, path string, mode uint32, flags int) error { } return err } - -func Eaccess(path string, mode uint32) error { - return faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/faccessat_solaris.go b/src/internal/syscall/unix/faccessat_solaris.go new file mode 100644 index 00000000..47e05fb2 --- /dev/null +++ b/src/internal/syscall/unix/faccessat_solaris.go @@ -0,0 +1,28 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "syscall" + "unsafe" +) + +//go:linkname procFaccessat libc_faccessat + +var procFaccessat uintptr + +func faccessat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procFaccessat)), 4, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) + if errno != 0 { + return errno + } + + return nil +} diff --git a/src/internal/syscall/unix/faccessat_syscall.go b/src/internal/syscall/unix/faccessat_syscall.go new file mode 100644 index 00000000..865e40b2 --- /dev/null +++ b/src/internal/syscall/unix/faccessat_syscall.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || linux + +package unix + +import "syscall" + +var faccessat = syscall.Faccessat diff --git a/src/internal/syscall/unix/getentropy_openbsd.go b/src/internal/syscall/unix/getentropy_openbsd.go deleted file mode 100644 index ad0914da..00000000 --- a/src/internal/syscall/unix/getentropy_openbsd.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && !mips64 - -package unix - -import _ "unsafe" // for linkname - -// GetEntropy calls the OpenBSD getentropy system call. -func GetEntropy(p []byte) error { - return getentropy(p) -} - -//go:linkname getentropy syscall.getentropy -func getentropy(p []byte) error diff --git a/src/internal/syscall/unix/getentropy_openbsd_mips64.go b/src/internal/syscall/unix/getentropy_openbsd_mips64.go deleted file mode 100644 index d5caa809..00000000 --- a/src/internal/syscall/unix/getentropy_openbsd_mips64.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import ( - "syscall" - "unsafe" -) - -// getentropy(2)'s syscall number, from /usr/src/sys/kern/syscalls.master -const entropyTrap uintptr = 7 - -// GetEntropy calls the OpenBSD getentropy system call. -func GetEntropy(p []byte) error { - _, _, errno := syscall.Syscall(entropyTrap, - uintptr(unsafe.Pointer(&p[0])), - uintptr(len(p)), - 0) - if errno != 0 { - return errno - } - return nil -} diff --git a/src/internal/syscall/unix/getrandom.go b/src/internal/syscall/unix/getrandom.go index e83f0cd6..db3e7ac0 100644 --- a/src/internal/syscall/unix/getrandom.go +++ b/src/internal/syscall/unix/getrandom.go @@ -12,6 +12,10 @@ import ( "unsafe" ) +//go:linkname vgetrandom runtime.vgetrandom +//go:noescape +func vgetrandom(p []byte, flags uint32) (ret int, supported bool) + var getrandomUnsupported atomic.Bool // GetRandomFlag is a flag supported by the getrandom system call. @@ -19,14 +23,18 @@ type GetRandomFlag uintptr // GetRandom calls the getrandom system call. func GetRandom(p []byte, flags GetRandomFlag) (n int, err error) { - if len(p) == 0 { - return 0, nil + ret, supported := vgetrandom(p, uint32(flags)) + if supported { + if ret < 0 { + return 0, syscall.Errno(-ret) + } + return ret, nil } if getrandomUnsupported.Load() { return 0, syscall.ENOSYS } r1, _, errno := syscall.Syscall(getrandomTrap, - uintptr(unsafe.Pointer(&p[0])), + uintptr(unsafe.Pointer(unsafe.SliceData(p))), uintptr(len(p)), uintptr(flags)) if errno != 0 { diff --git a/src/internal/syscall/unix/getrandom_linux_test.go b/src/internal/syscall/unix/getrandom_linux_test.go new file mode 100644 index 00000000..e1778c19 --- /dev/null +++ b/src/internal/syscall/unix/getrandom_linux_test.go @@ -0,0 +1,22 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix_test + +import ( + "internal/syscall/unix" + "testing" +) + +func BenchmarkParallelGetRandom(b *testing.B) { + b.SetBytes(4) + b.RunParallel(func(pb *testing.PB) { + var buf [4]byte + for pb.Next() { + if _, err := unix.GetRandom(buf[:], 0); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/src/internal/syscall/unix/kernel_version_freebsd.go b/src/internal/syscall/unix/kernel_version_freebsd.go new file mode 100644 index 00000000..ef9ee136 --- /dev/null +++ b/src/internal/syscall/unix/kernel_version_freebsd.go @@ -0,0 +1,48 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "sync" + "syscall" +) + +// KernelVersion returns major and minor kernel version numbers +// parsed from the syscall.Sysctl("kern.osrelease")'s value, +// or (0, 0) if the version can't be obtained or parsed. +func KernelVersion() (major, minor int) { + release, err := syscall.Sysctl("kern.osrelease") + if err != nil { + return 0, 0 + } + + parseNext := func() (n int) { + for i, c := range release { + if c == '.' { + release = release[i+1:] + return + } + if '0' <= c && c <= '9' { + n = n*10 + int(c-'0') + } + } + release = "" + return + } + + major = parseNext() + minor = parseNext() + + return +} + +// SupportCopyFileRange reports whether the kernel supports the copy_file_range(2). +// This function will examine both the kernel version and the availability of the system call. +var SupportCopyFileRange = sync.OnceValue(func() bool { + // The copy_file_range() function first appeared in FreeBSD 13.0. + major, _ := KernelVersion() + _, err := CopyFileRange(0, nil, 0, nil, 0, 0) + return major >= 13 && err != syscall.ENOSYS +}) diff --git a/src/internal/syscall/unix/kernel_version_freebsd_test.go b/src/internal/syscall/unix/kernel_version_freebsd_test.go new file mode 100644 index 00000000..694440e3 --- /dev/null +++ b/src/internal/syscall/unix/kernel_version_freebsd_test.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix_test + +import ( + "internal/syscall/unix" + "syscall" + "testing" +) + +func TestSupportCopyFileRange(t *testing.T) { + major, minor := unix.KernelVersion() + t.Logf("Running on FreeBSD %d.%d\n", major, minor) + + _, err := unix.CopyFileRange(0, nil, 0, nil, 0, 0) + want := err != syscall.ENOSYS + got := unix.SupportCopyFileRange() + if want != got { + t.Fatalf("SupportCopyFileRange, got %t; want %t", got, want) + } +} diff --git a/src/internal/syscall/unix/kernel_version_linux.go b/src/internal/syscall/unix/kernel_version_linux.go index 71e8aa4c..f3656288 100644 --- a/src/internal/syscall/unix/kernel_version_linux.go +++ b/src/internal/syscall/unix/kernel_version_linux.go @@ -8,11 +8,9 @@ import ( "syscall" ) -// KernelVersion returns major and minor kernel version numbers, parsed from -// the syscall.Uname's Release field, or 0, 0 if the version can't be obtained -// or parsed. -// -// Currently only implemented for Linux. +// KernelVersion returns major and minor kernel version numbers +// parsed from the syscall.Uname's Release field, or (0, 0) if +// the version can't be obtained or parsed. func KernelVersion() (major, minor int) { var uname syscall.Utsname if err := syscall.Uname(&uname); err != nil { diff --git a/src/internal/syscall/unix/kernel_version_other.go b/src/internal/syscall/unix/kernel_version_other.go index fc65c1c8..91c14b31 100644 --- a/src/internal/syscall/unix/kernel_version_other.go +++ b/src/internal/syscall/unix/kernel_version_other.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !linux && !solaris +//go:build !freebsd && !linux && !solaris package unix diff --git a/src/internal/syscall/unix/net_darwin.go b/src/internal/syscall/unix/net_darwin.go index bbaa94b0..7e8f2ac1 100644 --- a/src/internal/syscall/unix/net_darwin.go +++ b/src/internal/syscall/unix/net_darwin.go @@ -16,12 +16,13 @@ const ( AI_V4MAPPED = 0x800 AI_MASK = 0x1407 - EAI_AGAIN = 2 - EAI_NODATA = 7 - EAI_NONAME = 8 - EAI_SERVICE = 9 - EAI_SYSTEM = 11 - EAI_OVERFLOW = 14 + EAI_ADDRFAMILY = 1 + EAI_AGAIN = 2 + EAI_NODATA = 7 + EAI_NONAME = 8 + EAI_SERVICE = 9 + EAI_SYSTEM = 11 + EAI_OVERFLOW = 14 NI_NAMEREQD = 4 ) diff --git a/src/internal/syscall/unix/syscall.go b/src/internal/syscall/unix/syscall.go new file mode 100644 index 00000000..99805d98 --- /dev/null +++ b/src/internal/syscall/unix/syscall.go @@ -0,0 +1,8 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +// Single-word zero for use when we need a valid pointer to 0 bytes. +var _zero uintptr diff --git a/src/cmd/compile/internal/importer/testdata/alias.go b/src/internal/syscall/unix/sysnum_freebsd.go similarity index 75% rename from src/cmd/compile/internal/importer/testdata/alias.go rename to src/internal/syscall/unix/sysnum_freebsd.go index 51492fc9..2c811104 100644 --- a/src/cmd/compile/internal/importer/testdata/alias.go +++ b/src/internal/syscall/unix/sysnum_freebsd.go @@ -2,6 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package testdata +package unix -type A = int32 +const copyFileRangeTrap uintptr = 569 diff --git a/src/internal/syscall/unix/sysnum_linux_386.go b/src/internal/syscall/unix/sysnum_linux_386.go index be048bcf..c83beef7 100644 --- a/src/internal/syscall/unix/sysnum_linux_386.go +++ b/src/internal/syscall/unix/sysnum_linux_386.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 377 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_amd64.go b/src/internal/syscall/unix/sysnum_linux_amd64.go index 525de9cb..098e3320 100644 --- a/src/internal/syscall/unix/sysnum_linux_amd64.go +++ b/src/internal/syscall/unix/sysnum_linux_amd64.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 326 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_arm.go b/src/internal/syscall/unix/sysnum_linux_arm.go index b8038922..f0cd45f9 100644 --- a/src/internal/syscall/unix/sysnum_linux_arm.go +++ b/src/internal/syscall/unix/sysnum_linux_arm.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 391 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_generic.go b/src/internal/syscall/unix/sysnum_linux_generic.go index b06bf692..ec622cff 100644 --- a/src/internal/syscall/unix/sysnum_linux_generic.go +++ b/src/internal/syscall/unix/sysnum_linux_generic.go @@ -15,4 +15,5 @@ const ( copyFileRangeTrap uintptr = 285 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_mips64x.go b/src/internal/syscall/unix/sysnum_linux_mips64x.go index 8764f5dc..3875105d 100644 --- a/src/internal/syscall/unix/sysnum_linux_mips64x.go +++ b/src/internal/syscall/unix/sysnum_linux_mips64x.go @@ -11,4 +11,5 @@ const ( copyFileRangeTrap uintptr = 5320 pidfdSendSignalTrap uintptr = 5424 pidfdOpenTrap uintptr = 5434 + openat2Trap uintptr = 5437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_mipsx.go b/src/internal/syscall/unix/sysnum_linux_mipsx.go index 9b2e587b..bdd2fef2 100644 --- a/src/internal/syscall/unix/sysnum_linux_mipsx.go +++ b/src/internal/syscall/unix/sysnum_linux_mipsx.go @@ -11,4 +11,5 @@ const ( copyFileRangeTrap uintptr = 4360 pidfdSendSignalTrap uintptr = 4424 pidfdOpenTrap uintptr = 4434 + openat2Trap uintptr = 4437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_ppc64x.go b/src/internal/syscall/unix/sysnum_linux_ppc64x.go index 03e9c197..9291d58f 100644 --- a/src/internal/syscall/unix/sysnum_linux_ppc64x.go +++ b/src/internal/syscall/unix/sysnum_linux_ppc64x.go @@ -11,4 +11,5 @@ const ( copyFileRangeTrap uintptr = 379 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_s390x.go b/src/internal/syscall/unix/sysnum_linux_s390x.go index c6e3e02e..65a82d1c 100644 --- a/src/internal/syscall/unix/sysnum_linux_s390x.go +++ b/src/internal/syscall/unix/sysnum_linux_s390x.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 375 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/waitid_linux.go b/src/internal/syscall/unix/waitid_linux.go new file mode 100644 index 00000000..240a7f74 --- /dev/null +++ b/src/internal/syscall/unix/waitid_linux.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "syscall" + "unsafe" +) + +const ( + P_PID = 1 + P_PIDFD = 3 +) + +func Waitid(idType int, id int, info *SiginfoChild, options int, rusage *syscall.Rusage) error { + _, _, errno := syscall.Syscall6(syscall.SYS_WAITID, uintptr(idType), uintptr(id), uintptr(unsafe.Pointer(info)), uintptr(options), uintptr(unsafe.Pointer(rusage)), 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/windows/at_windows.go b/src/internal/syscall/windows/at_windows.go new file mode 100644 index 00000000..18429773 --- /dev/null +++ b/src/internal/syscall/windows/at_windows.go @@ -0,0 +1,252 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import ( + "syscall" + "unsafe" +) + +// Openat flags not supported by syscall.Open. +// +// These are invented values. +// +// When adding a new flag here, add an unexported version to +// the set of invented O_ values in syscall/types_windows.go +// to avoid overlap. +const ( + O_DIRECTORY = 0x100000 // target must be a directory + O_NOFOLLOW_ANY = 0x20000000 // disallow symlinks anywhere in the path + O_OPEN_REPARSE = 0x40000000 // FILE_OPEN_REPARSE_POINT, used by Lstat +) + +func Openat(dirfd syscall.Handle, name string, flag int, perm uint32) (_ syscall.Handle, e1 error) { + if len(name) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + + var access, options uint32 + switch flag & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + // FILE_GENERIC_READ includes FILE_LIST_DIRECTORY. + access = FILE_GENERIC_READ + case syscall.O_WRONLY: + access = FILE_GENERIC_WRITE + options |= FILE_NON_DIRECTORY_FILE + case syscall.O_RDWR: + access = FILE_GENERIC_READ | FILE_GENERIC_WRITE + options |= FILE_NON_DIRECTORY_FILE + default: + // Stat opens files without requesting read or write permissions, + // but we still need to request SYNCHRONIZE. + access = SYNCHRONIZE + } + if flag&syscall.O_CREAT != 0 { + access |= FILE_GENERIC_WRITE + } + if flag&syscall.O_APPEND != 0 { + access |= FILE_APPEND_DATA + // Remove FILE_WRITE_DATA access unless O_TRUNC is set, + // in which case we need it to truncate the file. + if flag&syscall.O_TRUNC == 0 { + access &^= FILE_WRITE_DATA + } + } + if flag&O_DIRECTORY != 0 { + options |= FILE_DIRECTORY_FILE + access |= FILE_LIST_DIRECTORY + } + if flag&syscall.O_SYNC != 0 { + options |= FILE_WRITE_THROUGH + } + // Allow File.Stat. + access |= STANDARD_RIGHTS_READ | FILE_READ_ATTRIBUTES | FILE_READ_EA + + objAttrs := &OBJECT_ATTRIBUTES{} + if flag&O_NOFOLLOW_ANY != 0 { + objAttrs.Attributes |= OBJ_DONT_REPARSE + } + if flag&syscall.O_CLOEXEC == 0 { + objAttrs.Attributes |= OBJ_INHERIT + } + if err := objAttrs.init(dirfd, name); err != nil { + return syscall.InvalidHandle, err + } + + if flag&O_OPEN_REPARSE != 0 { + options |= FILE_OPEN_REPARSE_POINT + } + + // We don't use FILE_OVERWRITE/FILE_OVERWRITE_IF, because when opening + // a file with FILE_ATTRIBUTE_READONLY these will replace an existing + // file with a new, read-only one. + // + // Instead, we ftruncate the file after opening when O_TRUNC is set. + var disposition uint32 + switch { + case flag&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + disposition = FILE_CREATE + case flag&syscall.O_CREAT == syscall.O_CREAT: + disposition = FILE_OPEN_IF + default: + disposition = FILE_OPEN + } + + fileAttrs := uint32(FILE_ATTRIBUTE_NORMAL) + if perm&syscall.S_IWRITE == 0 { + fileAttrs = FILE_ATTRIBUTE_READONLY + } + + var h syscall.Handle + err := NtCreateFile( + &h, + SYNCHRONIZE|access, + objAttrs, + &IO_STATUS_BLOCK{}, + nil, + fileAttrs, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + disposition, + FILE_SYNCHRONOUS_IO_NONALERT|FILE_OPEN_FOR_BACKUP_INTENT|options, + 0, + 0, + ) + if err != nil { + return h, ntCreateFileError(err, flag) + } + + if flag&syscall.O_TRUNC != 0 { + err = syscall.Ftruncate(h, 0) + if err != nil { + syscall.CloseHandle(h) + return syscall.InvalidHandle, err + } + } + + return h, nil +} + +// ntCreateFileError maps error returns from NTCreateFile to user-visible errors. +func ntCreateFileError(err error, flag int) error { + s, ok := err.(NTStatus) + if !ok { + // Shouldn't really be possible, NtCreateFile always returns NTStatus. + return err + } + switch s { + case STATUS_REPARSE_POINT_ENCOUNTERED: + return syscall.ELOOP + case STATUS_NOT_A_DIRECTORY: + // ENOTDIR is the errno returned by open when O_DIRECTORY is specified + // and the target is not a directory. + // + // NtCreateFile can return STATUS_NOT_A_DIRECTORY under other circumstances, + // such as when opening "file/" where "file" is not a directory. + // (This might be Windows version dependent.) + // + // Only map STATUS_NOT_A_DIRECTORY to ENOTDIR when O_DIRECTORY is specified. + if flag&O_DIRECTORY != 0 { + return syscall.ENOTDIR + } + case STATUS_FILE_IS_A_DIRECTORY: + return syscall.EISDIR + } + return s.Errno() +} + +func Mkdirat(dirfd syscall.Handle, name string, mode uint32) error { + objAttrs := &OBJECT_ATTRIBUTES{} + if err := objAttrs.init(dirfd, name); err != nil { + return err + } + var h syscall.Handle + err := NtCreateFile( + &h, + FILE_GENERIC_READ, + objAttrs, + &IO_STATUS_BLOCK{}, + nil, + syscall.FILE_ATTRIBUTE_NORMAL, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + FILE_CREATE, + FILE_DIRECTORY_FILE, + 0, + 0, + ) + if err != nil { + return ntCreateFileError(err, 0) + } + syscall.CloseHandle(h) + return nil +} + +func Deleteat(dirfd syscall.Handle, name string) error { + objAttrs := &OBJECT_ATTRIBUTES{} + if err := objAttrs.init(dirfd, name); err != nil { + return err + } + var h syscall.Handle + err := NtOpenFile( + &h, + DELETE, + objAttrs, + &IO_STATUS_BLOCK{}, + FILE_SHARE_DELETE|FILE_SHARE_READ|FILE_SHARE_WRITE, + FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT, + ) + if err != nil { + return ntCreateFileError(err, 0) + } + defer syscall.CloseHandle(h) + + const ( + FileDispositionInformation = 13 + FileDispositionInformationEx = 64 + ) + + // First, attempt to delete the file using POSIX semantics + // (which permit a file to be deleted while it is still open). + // This matches the behavior of DeleteFileW. + err = NtSetInformationFile( + h, + &IO_STATUS_BLOCK{}, + uintptr(unsafe.Pointer(&FILE_DISPOSITION_INFORMATION_EX{ + Flags: FILE_DISPOSITION_DELETE | + FILE_DISPOSITION_FORCE_IMAGE_SECTION_CHECK | + FILE_DISPOSITION_POSIX_SEMANTICS | + // This differs from DeleteFileW, but matches os.Remove's + // behavior on Unix platforms of permitting deletion of + // read-only files. + FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE, + })), + uint32(unsafe.Sizeof(FILE_DISPOSITION_INFORMATION_EX{})), + FileDispositionInformationEx, + ) + switch err { + case nil: + return nil + case STATUS_CANNOT_DELETE, STATUS_DIRECTORY_NOT_EMPTY: + return err.(NTStatus).Errno() + } + + // If the prior deletion failed, the filesystem either doesn't support + // POSIX semantics (for example, FAT), or hasn't implemented + // FILE_DISPOSITION_INFORMATION_EX. + // + // Try again. + err = NtSetInformationFile( + h, + &IO_STATUS_BLOCK{}, + uintptr(unsafe.Pointer(&FILE_DISPOSITION_INFORMATION{ + DeleteFile: true, + })), + uint32(unsafe.Sizeof(FILE_DISPOSITION_INFORMATION{})), + FileDispositionInformation, + ) + if st, ok := err.(NTStatus); ok { + return st.Errno() + } + return err +} diff --git a/src/internal/syscall/windows/at_windows_test.go b/src/internal/syscall/windows/at_windows_test.go new file mode 100644 index 00000000..7da9ecf0 --- /dev/null +++ b/src/internal/syscall/windows/at_windows_test.go @@ -0,0 +1,58 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows_test + +import ( + "internal/syscall/windows" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestOpen(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + file := filepath.Join(dir, "a") + f, err := os.Create(file) + if err != nil { + t.Fatal(err) + } + f.Close() + + tests := []struct { + path string + flag int + err error + }{ + {dir, syscall.O_RDONLY, nil}, + {dir, syscall.O_CREAT, nil}, + {dir, syscall.O_RDONLY | syscall.O_CREAT, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE | os.O_TRUNC, nil}, + {dir, syscall.O_RDONLY | syscall.O_TRUNC, syscall.ERROR_ACCESS_DENIED}, + {dir, syscall.O_WRONLY | syscall.O_RDWR, nil}, // TODO: syscall.Open returns EISDIR here, we should reconcile this + {dir, syscall.O_WRONLY, syscall.EISDIR}, + {dir, syscall.O_RDWR, syscall.EISDIR}, + } + for i, tt := range tests { + dir := filepath.Dir(tt.path) + dirfd, err := syscall.Open(dir, syscall.O_RDONLY, 0) + if err != nil { + t.Error(err) + continue + } + base := filepath.Base(tt.path) + h, err := windows.Openat(dirfd, base, tt.flag, 0o660) + syscall.CloseHandle(dirfd) + if err == nil { + syscall.CloseHandle(h) + } + if err != tt.err { + t.Errorf("%d: Open got %q, want %q", i, err, tt.err) + } + } +} diff --git a/src/internal/syscall/windows/security_windows.go b/src/internal/syscall/windows/security_windows.go index c8c5cbed..017e25aa 100644 --- a/src/internal/syscall/windows/security_windows.go +++ b/src/internal/syscall/windows/security_windows.go @@ -5,6 +5,7 @@ package windows import ( + "runtime" "syscall" "unsafe" ) @@ -18,6 +19,8 @@ const ( //sys ImpersonateSelf(impersonationlevel uint32) (err error) = advapi32.ImpersonateSelf //sys RevertToSelf() (err error) = advapi32.RevertToSelf +//sys ImpersonateLoggedOnUser(token syscall.Token) (err error) = advapi32.ImpersonateLoggedOnUser +//sys LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32, token *syscall.Token) (err error) = advapi32.LogonUserW const ( TOKEN_ADJUST_PRIVILEGES = 0x0020 @@ -93,6 +96,26 @@ type LocalGroupUserInfo0 struct { Name *uint16 } +const ( + NERR_UserNotFound syscall.Errno = 2221 + NERR_UserExists syscall.Errno = 2224 +) + +const ( + USER_PRIV_USER = 1 +) + +type UserInfo1 struct { + Name *uint16 + Password *uint16 + PasswordAge uint32 + Priv uint32 + HomeDir *uint16 + Comment *uint16 + Flags uint32 + ScriptPath *uint16 +} + type UserInfo4 struct { Name *uint16 Password *uint16 @@ -125,6 +148,8 @@ type UserInfo4 struct { PasswordExpired uint32 } +//sys NetUserAdd(serverName *uint16, level uint32, buf *byte, parmErr *uint32) (neterr error) = netapi32.NetUserAdd +//sys NetUserDel(serverName *uint16, userName *uint16) (neterr error) = netapi32.NetUserDel //sys NetUserGetLocalGroups(serverName *uint16, userName *uint16, level uint32, flags uint32, buf **byte, prefMaxLen uint32, entriesRead *uint32, totalEntries *uint32) (neterr error) = netapi32.NetUserGetLocalGroups // GetSystemDirectory retrieves the path to current location of the system @@ -132,3 +157,108 @@ type UserInfo4 struct { // //go:linkname GetSystemDirectory func GetSystemDirectory() string // Implemented in runtime package. + +// GetUserName retrieves the user name of the current thread +// in the specified format. +func GetUserName(format uint32) (string, error) { + n := uint32(50) + for { + b := make([]uint16, n) + e := syscall.GetUserNameEx(format, &b[0], &n) + if e == nil { + return syscall.UTF16ToString(b[:n]), nil + } + if e != syscall.ERROR_MORE_DATA { + return "", e + } + if n <= uint32(len(b)) { + return "", e + } + } +} + +// getTokenInfo retrieves a specified type of information about an access token. +func getTokenInfo(t syscall.Token, class uint32, initSize int) (unsafe.Pointer, error) { + n := uint32(initSize) + for { + b := make([]byte, n) + e := syscall.GetTokenInformation(t, class, &b[0], uint32(len(b)), &n) + if e == nil { + return unsafe.Pointer(&b[0]), nil + } + if e != syscall.ERROR_INSUFFICIENT_BUFFER { + return nil, e + } + if n <= uint32(len(b)) { + return nil, e + } + } +} + +type TOKEN_GROUPS struct { + GroupCount uint32 + Groups [1]SID_AND_ATTRIBUTES +} + +func (g *TOKEN_GROUPS) AllGroups() []SID_AND_ATTRIBUTES { + return (*[(1 << 28) - 1]SID_AND_ATTRIBUTES)(unsafe.Pointer(&g.Groups[0]))[:g.GroupCount:g.GroupCount] +} + +func GetTokenGroups(t syscall.Token) (*TOKEN_GROUPS, error) { + i, e := getTokenInfo(t, syscall.TokenGroups, 50) + if e != nil { + return nil, e + } + return (*TOKEN_GROUPS)(i), nil +} + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid_identifier_authority +type SID_IDENTIFIER_AUTHORITY struct { + Value [6]byte +} + +const ( + SID_REVISION = 1 + // https://learn.microsoft.com/en-us/windows/win32/services/localsystem-account + SECURITY_LOCAL_SYSTEM_RID = 18 + // https://learn.microsoft.com/en-us/windows/win32/services/localservice-account + SECURITY_LOCAL_SERVICE_RID = 19 + // https://learn.microsoft.com/en-us/windows/win32/services/networkservice-account + SECURITY_NETWORK_SERVICE_RID = 20 +) + +var SECURITY_NT_AUTHORITY = SID_IDENTIFIER_AUTHORITY{ + Value: [6]byte{0, 0, 0, 0, 0, 5}, +} + +//sys IsValidSid(sid *syscall.SID) (valid bool) = advapi32.IsValidSid +//sys getSidIdentifierAuthority(sid *syscall.SID) (idauth uintptr) = advapi32.GetSidIdentifierAuthority +//sys getSidSubAuthority(sid *syscall.SID, subAuthorityIdx uint32) (subAuth uintptr) = advapi32.GetSidSubAuthority +//sys getSidSubAuthorityCount(sid *syscall.SID) (count uintptr) = advapi32.GetSidSubAuthorityCount + +// The following GetSid* functions are marked as //go:nocheckptr because checkptr +// instrumentation can't see that the pointer returned by the syscall is pointing +// into the sid's memory, which is normally allocated on the Go heap. Therefore, +// the checkptr instrumentation would incorrectly flag the pointer dereference +// as pointing to an invalid allocation. +// Also, use runtime.KeepAlive to ensure that the sid is not garbage collected +// before the GetSid* functions return, as the Go GC is not aware that the +// pointers returned by the syscall are pointing into the sid's memory. + +//go:nocheckptr +func GetSidIdentifierAuthority(sid *syscall.SID) SID_IDENTIFIER_AUTHORITY { + defer runtime.KeepAlive(sid) + return *(*SID_IDENTIFIER_AUTHORITY)(unsafe.Pointer(getSidIdentifierAuthority(sid))) +} + +//go:nocheckptr +func GetSidSubAuthority(sid *syscall.SID, subAuthorityIdx uint32) uint32 { + defer runtime.KeepAlive(sid) + return *(*uint32)(unsafe.Pointer(getSidSubAuthority(sid, subAuthorityIdx))) +} + +//go:nocheckptr +func GetSidSubAuthorityCount(sid *syscall.SID) uint8 { + defer runtime.KeepAlive(sid) + return *(*uint8)(unsafe.Pointer(getSidSubAuthorityCount(sid))) +} diff --git a/src/internal/syscall/windows/string_windows.go b/src/internal/syscall/windows/string_windows.go new file mode 100644 index 00000000..eb6893d7 --- /dev/null +++ b/src/internal/syscall/windows/string_windows.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import "syscall" + +// NTUnicodeString is a UTF-16 string for NT native APIs, corresponding to UNICODE_STRING. +type NTUnicodeString struct { + Length uint16 + MaximumLength uint16 + Buffer *uint16 +} + +// NewNTUnicodeString returns a new NTUnicodeString structure for use with native +// NT APIs that work over the NTUnicodeString type. Note that most Windows APIs +// do not use NTUnicodeString, and instead UTF16PtrFromString should be used for +// the more common *uint16 string type. +func NewNTUnicodeString(s string) (*NTUnicodeString, error) { + s16, err := syscall.UTF16FromString(s) + if err != nil { + return nil, err + } + n := uint16(len(s16) * 2) + // https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdmsec/nf-wdmsec-wdmlibrtlinitunicodestringex + return &NTUnicodeString{ + Length: n - 2, // subtract 2 bytes for the NUL terminator + MaximumLength: n, + Buffer: &s16[0], + }, nil +} diff --git a/src/internal/syscall/windows/syscall_windows.go b/src/internal/syscall/windows/syscall_windows.go index b0b5a648..715d0724 100644 --- a/src/internal/syscall/windows/syscall_windows.go +++ b/src/internal/syscall/windows/syscall_windows.go @@ -39,7 +39,9 @@ const ( ERROR_CALL_NOT_IMPLEMENTED syscall.Errno = 120 ERROR_INVALID_NAME syscall.Errno = 123 ERROR_LOCK_FAILED syscall.Errno = 167 + ERROR_NO_TOKEN syscall.Errno = 1008 ERROR_NO_UNICODE_TRANSLATION syscall.Errno = 1113 + ERROR_CANT_ACCESS_FILE syscall.Errno = 1920 ) const ( @@ -499,3 +501,37 @@ func QueryPerformanceCounter() int64 // Implemented in runtime package. // //go:linkname QueryPerformanceFrequency func QueryPerformanceFrequency() int64 // Implemented in runtime package. + +//sys GetModuleHandle(modulename *uint16) (handle syscall.Handle, err error) = kernel32.GetModuleHandleW + +// NTStatus corresponds with NTSTATUS, error values returned by ntdll.dll and +// other native functions. +type NTStatus uint32 + +func (s NTStatus) Errno() syscall.Errno { + return rtlNtStatusToDosErrorNoTeb(s) +} + +func langID(pri, sub uint16) uint32 { return uint32(sub)<<10 | uint32(pri) } + +func (s NTStatus) Error() string { + return s.Errno().Error() +} + +// x/sys/windows/mkerrors.bash can generate a complete list of NTStatus codes. +// +// At the moment, we only need a couple, so just put them here manually. +// If this list starts getting long, we should consider generating the full set. +const ( + STATUS_FILE_IS_A_DIRECTORY NTStatus = 0xC00000BA + STATUS_DIRECTORY_NOT_EMPTY NTStatus = 0xC0000101 + STATUS_NOT_A_DIRECTORY NTStatus = 0xC0000103 + STATUS_CANNOT_DELETE NTStatus = 0xC0000121 + STATUS_REPARSE_POINT_ENCOUNTERED NTStatus = 0xC000050B +) + +// NT Native APIs +//sys NtCreateFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, allocationSize *int64, attributes uint32, share uint32, disposition uint32, options uint32, eabuffer uintptr, ealength uint32) (ntstatus error) = ntdll.NtCreateFile +//sys NtOpenFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, share uint32, options uint32) (ntstatus error) = ntdll.NtOpenFile +//sys rtlNtStatusToDosErrorNoTeb(ntstatus NTStatus) (ret syscall.Errno) = ntdll.RtlNtStatusToDosErrorNoTeb +//sys NtSetInformationFile(handle syscall.Handle, iosb *IO_STATUS_BLOCK, inBuffer uintptr, inBufferLen uint32, class uint32) (ntstatus error) = ntdll.NtSetInformationFile diff --git a/src/internal/syscall/windows/types_windows.go b/src/internal/syscall/windows/types_windows.go index 126e07b8..6ae37aff 100644 --- a/src/internal/syscall/windows/types_windows.go +++ b/src/internal/syscall/windows/types_windows.go @@ -4,9 +4,215 @@ package windows +import ( + "syscall" + "unsafe" +) + // Socket related. const ( TCP_KEEPIDLE = 0x03 TCP_KEEPCNT = 0x10 TCP_KEEPINTVL = 0x11 ) + +const ( + FILE_READ_DATA = 0x00000001 + FILE_READ_ATTRIBUTES = 0x00000080 + FILE_READ_EA = 0x00000008 + FILE_WRITE_DATA = 0x00000002 + FILE_WRITE_ATTRIBUTES = 0x00000100 + FILE_WRITE_EA = 0x00000010 + FILE_APPEND_DATA = 0x00000004 + FILE_EXECUTE = 0x00000020 + + FILE_GENERIC_READ = STANDARD_RIGHTS_READ | FILE_READ_DATA | FILE_READ_ATTRIBUTES | FILE_READ_EA | SYNCHRONIZE + FILE_GENERIC_WRITE = STANDARD_RIGHTS_WRITE | FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA | FILE_APPEND_DATA | SYNCHRONIZE + FILE_GENERIC_EXECUTE = STANDARD_RIGHTS_EXECUTE | FILE_READ_ATTRIBUTES | FILE_EXECUTE | SYNCHRONIZE + + FILE_LIST_DIRECTORY = 0x00000001 + FILE_TRAVERSE = 0x00000020 + + FILE_SHARE_READ = 0x00000001 + FILE_SHARE_WRITE = 0x00000002 + FILE_SHARE_DELETE = 0x00000004 + FILE_ATTRIBUTE_READONLY = 0x00000001 + FILE_ATTRIBUTE_HIDDEN = 0x00000002 + FILE_ATTRIBUTE_SYSTEM = 0x00000004 + FILE_ATTRIBUTE_DIRECTORY = 0x00000010 + FILE_ATTRIBUTE_ARCHIVE = 0x00000020 + FILE_ATTRIBUTE_DEVICE = 0x00000040 + FILE_ATTRIBUTE_NORMAL = 0x00000080 + FILE_ATTRIBUTE_TEMPORARY = 0x00000100 + FILE_ATTRIBUTE_SPARSE_FILE = 0x00000200 + FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400 + FILE_ATTRIBUTE_COMPRESSED = 0x00000800 + FILE_ATTRIBUTE_OFFLINE = 0x00001000 + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x00002000 + FILE_ATTRIBUTE_ENCRYPTED = 0x00004000 + FILE_ATTRIBUTE_INTEGRITY_STREAM = 0x00008000 + FILE_ATTRIBUTE_VIRTUAL = 0x00010000 + FILE_ATTRIBUTE_NO_SCRUB_DATA = 0x00020000 + FILE_ATTRIBUTE_RECALL_ON_OPEN = 0x00040000 + FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 0x00400000 + + INVALID_FILE_ATTRIBUTES = 0xffffffff +) + +// https://learn.microsoft.com/en-us/windows/win32/secauthz/access-mask +type ACCESS_MASK uint32 + +// Constants for type ACCESS_MASK +const ( + DELETE = 0x00010000 + READ_CONTROL = 0x00020000 + WRITE_DAC = 0x00040000 + WRITE_OWNER = 0x00080000 + SYNCHRONIZE = 0x00100000 + STANDARD_RIGHTS_REQUIRED = 0x000F0000 + STANDARD_RIGHTS_READ = READ_CONTROL + STANDARD_RIGHTS_WRITE = READ_CONTROL + STANDARD_RIGHTS_EXECUTE = READ_CONTROL + STANDARD_RIGHTS_ALL = 0x001F0000 + SPECIFIC_RIGHTS_ALL = 0x0000FFFF + ACCESS_SYSTEM_SECURITY = 0x01000000 + MAXIMUM_ALLOWED = 0x02000000 + GENERIC_READ = 0x80000000 + GENERIC_WRITE = 0x40000000 + GENERIC_EXECUTE = 0x20000000 + GENERIC_ALL = 0x10000000 +) + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_acl +type ACL struct { + AclRevision byte + Sbz1 byte + AclSize uint16 + AceCount uint16 + Sbz2 uint16 +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_io_status_block +type IO_STATUS_BLOCK struct { + Status NTStatus + Information uintptr +} + +// https://learn.microsoft.com/en-us/windows/win32/api/ntdef/ns-ntdef-_object_attributes +type OBJECT_ATTRIBUTES struct { + Length uint32 + RootDirectory syscall.Handle + ObjectName *NTUnicodeString + Attributes uint32 + SecurityDescriptor *SECURITY_DESCRIPTOR + SecurityQoS *SECURITY_QUALITY_OF_SERVICE +} + +// init sets o's RootDirectory, ObjectName, and Length. +func (o *OBJECT_ATTRIBUTES) init(root syscall.Handle, name string) error { + if name == "." { + name = "" + } + objectName, err := NewNTUnicodeString(name) + if err != nil { + return err + } + o.ObjectName = objectName + if root != syscall.InvalidHandle { + o.RootDirectory = root + } + o.Length = uint32(unsafe.Sizeof(*o)) + return nil +} + +// Values for the Attributes member of OBJECT_ATTRIBUTES. +const ( + OBJ_INHERIT = 0x00000002 + OBJ_PERMANENT = 0x00000010 + OBJ_EXCLUSIVE = 0x00000020 + OBJ_CASE_INSENSITIVE = 0x00000040 + OBJ_OPENIF = 0x00000080 + OBJ_OPENLINK = 0x00000100 + OBJ_KERNEL_HANDLE = 0x00000200 + OBJ_FORCE_ACCESS_CHECK = 0x00000400 + OBJ_IGNORE_IMPERSONATED_DEVICEMAP = 0x00000800 + OBJ_DONT_REPARSE = 0x00001000 + OBJ_VALID_ATTRIBUTES = 0x00001FF2 +) + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_security_descriptor +type SECURITY_DESCRIPTOR struct { + revision byte + sbz1 byte + control SECURITY_DESCRIPTOR_CONTROL + owner *syscall.SID + group *syscall.SID + sacl *ACL + dacl *ACL +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ifs/security-descriptor-control +type SECURITY_DESCRIPTOR_CONTROL uint16 + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-security_quality_of_service +type SECURITY_QUALITY_OF_SERVICE struct { + Length uint32 + ImpersonationLevel uint32 // type SECURITY_IMPERSONATION_LEVEL + ContextTrackingMode byte // type SECURITY_CONTEXT_TRACKING_MODE + EffectiveOnly byte +} + +const ( + // CreateDisposition flags for NtCreateFile and NtCreateNamedPipeFile. + FILE_SUPERSEDE = 0x00000000 + FILE_OPEN = 0x00000001 + FILE_CREATE = 0x00000002 + FILE_OPEN_IF = 0x00000003 + FILE_OVERWRITE = 0x00000004 + FILE_OVERWRITE_IF = 0x00000005 + FILE_MAXIMUM_DISPOSITION = 0x00000005 + + // CreateOptions flags for NtCreateFile and NtCreateNamedPipeFile. + FILE_DIRECTORY_FILE = 0x00000001 + FILE_WRITE_THROUGH = 0x00000002 + FILE_SEQUENTIAL_ONLY = 0x00000004 + FILE_NO_INTERMEDIATE_BUFFERING = 0x00000008 + FILE_SYNCHRONOUS_IO_ALERT = 0x00000010 + FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020 + FILE_NON_DIRECTORY_FILE = 0x00000040 + FILE_CREATE_TREE_CONNECTION = 0x00000080 + FILE_COMPLETE_IF_OPLOCKED = 0x00000100 + FILE_NO_EA_KNOWLEDGE = 0x00000200 + FILE_OPEN_REMOTE_INSTANCE = 0x00000400 + FILE_RANDOM_ACCESS = 0x00000800 + FILE_DELETE_ON_CLOSE = 0x00001000 + FILE_OPEN_BY_FILE_ID = 0x00002000 + FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000 + FILE_NO_COMPRESSION = 0x00008000 + FILE_OPEN_REQUIRING_OPLOCK = 0x00010000 + FILE_DISALLOW_EXCLUSIVE = 0x00020000 + FILE_RESERVE_OPFILTER = 0x00100000 + FILE_OPEN_REPARSE_POINT = 0x00200000 + FILE_OPEN_NO_RECALL = 0x00400000 + FILE_OPEN_FOR_FREE_SPACE_QUERY = 0x00800000 +) + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntddk/ns-ntddk-_file_disposition_information +type FILE_DISPOSITION_INFORMATION struct { + DeleteFile bool +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntddk/ns-ntddk-_file_disposition_information_ex +type FILE_DISPOSITION_INFORMATION_EX struct { + Flags uint32 +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntddk/ns-ntddk-_file_disposition_information_ex +const ( + FILE_DISPOSITION_DO_NOT_DELETE = 0x00000000 + FILE_DISPOSITION_DELETE = 0x00000001 + FILE_DISPOSITION_POSIX_SEMANTICS = 0x00000002 + FILE_DISPOSITION_FORCE_IMAGE_SECTION_CHECK = 0x00000004 + FILE_DISPOSITION_ON_CLOSE = 0x00000008 + FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE = 0x00000010 +) diff --git a/src/internal/syscall/windows/version_windows.go b/src/internal/syscall/windows/version_windows.go index ff21fc59..cb5f6ba6 100644 --- a/src/internal/syscall/windows/version_windows.go +++ b/src/internal/syscall/windows/version_windows.go @@ -24,9 +24,9 @@ type _OSVERSIONINFOW struct { // According to documentation, RtlGetVersion function always succeeds. //sys rtlGetVersion(info *_OSVERSIONINFOW) = ntdll.RtlGetVersion -// version retrieves the major, minor, and build version numbers +// Version retrieves the major, minor, and build version numbers // of the current Windows OS from the RtlGetVersion API. -func version() (major, minor, build uint32) { +func Version() (major, minor, build uint32) { info := _OSVERSIONINFOW{} info.osVersionInfoSize = uint32(unsafe.Sizeof(info)) rtlGetVersion(&info) @@ -43,7 +43,7 @@ var initTCPKeepAlive = sync.OnceFunc(func() { s, err := WSASocket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP, nil, 0, WSA_FLAG_NO_HANDLE_INHERIT) if err != nil { // Fallback to checking the Windows version. - major, _, build := version() + major, _, build := Version() supportTCPKeepAliveIdle = major >= 10 && build >= 16299 supportTCPKeepAliveInterval = major >= 10 && build >= 16299 supportTCPKeepAliveCount = major >= 10 && build >= 15063 @@ -59,7 +59,7 @@ var initTCPKeepAlive = sync.OnceFunc(func() { supportTCPKeepAliveCount = optSupported(TCP_KEEPCNT) }) -// SupportTCPKeepAliveInterval indicates whether TCP_KEEPIDLE is supported. +// SupportTCPKeepAliveIdle indicates whether TCP_KEEPIDLE is supported. // The minimal requirement is Windows 10.0.16299. func SupportTCPKeepAliveIdle() bool { initTCPKeepAlive() @@ -85,7 +85,7 @@ func SupportTCPKeepAliveCount() bool { // Windows version supports the TCP_INITIAL_RTO_NO_SYN_RETRANSMISSIONS. // The minimal requirement is Windows 10.0.16299. var SupportTCPInitialRTONoSYNRetransmissions = sync.OnceValue(func() bool { - major, _, build := version() + major, _, build := Version() return major >= 10 && build >= 16299 }) diff --git a/src/internal/syscall/windows/zsyscall_windows.go b/src/internal/syscall/windows/zsyscall_windows.go index 062641cd..9a096cfc 100644 --- a/src/internal/syscall/windows/zsyscall_windows.go +++ b/src/internal/syscall/windows/zsyscall_windows.go @@ -48,7 +48,13 @@ var ( procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges") procDuplicateTokenEx = modadvapi32.NewProc("DuplicateTokenEx") + procGetSidIdentifierAuthority = modadvapi32.NewProc("GetSidIdentifierAuthority") + procGetSidSubAuthority = modadvapi32.NewProc("GetSidSubAuthority") + procGetSidSubAuthorityCount = modadvapi32.NewProc("GetSidSubAuthorityCount") + procImpersonateLoggedOnUser = modadvapi32.NewProc("ImpersonateLoggedOnUser") procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf") + procIsValidSid = modadvapi32.NewProc("IsValidSid") + procLogonUserW = modadvapi32.NewProc("LogonUserW") procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW") procOpenSCManagerW = modadvapi32.NewProc("OpenSCManagerW") procOpenServiceW = modadvapi32.NewProc("OpenServiceW") @@ -66,6 +72,7 @@ var ( procGetFileInformationByHandleEx = modkernel32.NewProc("GetFileInformationByHandleEx") procGetFinalPathNameByHandleW = modkernel32.NewProc("GetFinalPathNameByHandleW") procGetModuleFileNameW = modkernel32.NewProc("GetModuleFileNameW") + procGetModuleHandleW = modkernel32.NewProc("GetModuleHandleW") procGetTempPath2W = modkernel32.NewProc("GetTempPath2W") procGetVolumeInformationByHandleW = modkernel32.NewProc("GetVolumeInformationByHandleW") procGetVolumeNameForVolumeMountPointW = modkernel32.NewProc("GetVolumeNameForVolumeMountPointW") @@ -81,8 +88,14 @@ var ( procVirtualQuery = modkernel32.NewProc("VirtualQuery") procNetShareAdd = modnetapi32.NewProc("NetShareAdd") procNetShareDel = modnetapi32.NewProc("NetShareDel") + procNetUserAdd = modnetapi32.NewProc("NetUserAdd") + procNetUserDel = modnetapi32.NewProc("NetUserDel") procNetUserGetLocalGroups = modnetapi32.NewProc("NetUserGetLocalGroups") + procNtCreateFile = modntdll.NewProc("NtCreateFile") + procNtOpenFile = modntdll.NewProc("NtOpenFile") + procNtSetInformationFile = modntdll.NewProc("NtSetInformationFile") procRtlGetVersion = modntdll.NewProc("RtlGetVersion") + procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb") procGetProcessMemoryInfo = modpsapi.NewProc("GetProcessMemoryInfo") procCreateEnvironmentBlock = moduserenv.NewProc("CreateEnvironmentBlock") procDestroyEnvironmentBlock = moduserenv.NewProc("DestroyEnvironmentBlock") @@ -112,6 +125,32 @@ func DuplicateTokenEx(hExistingToken syscall.Token, dwDesiredAccess uint32, lpTo return } +func getSidIdentifierAuthority(sid *syscall.SID) (idauth uintptr) { + r0, _, _ := syscall.Syscall(procGetSidIdentifierAuthority.Addr(), 1, uintptr(unsafe.Pointer(sid)), 0, 0) + idauth = uintptr(r0) + return +} + +func getSidSubAuthority(sid *syscall.SID, subAuthorityIdx uint32) (subAuth uintptr) { + r0, _, _ := syscall.Syscall(procGetSidSubAuthority.Addr(), 2, uintptr(unsafe.Pointer(sid)), uintptr(subAuthorityIdx), 0) + subAuth = uintptr(r0) + return +} + +func getSidSubAuthorityCount(sid *syscall.SID) (count uintptr) { + r0, _, _ := syscall.Syscall(procGetSidSubAuthorityCount.Addr(), 1, uintptr(unsafe.Pointer(sid)), 0, 0) + count = uintptr(r0) + return +} + +func ImpersonateLoggedOnUser(token syscall.Token) (err error) { + r1, _, e1 := syscall.Syscall(procImpersonateLoggedOnUser.Addr(), 1, uintptr(token), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func ImpersonateSelf(impersonationlevel uint32) (err error) { r1, _, e1 := syscall.Syscall(procImpersonateSelf.Addr(), 1, uintptr(impersonationlevel), 0, 0) if r1 == 0 { @@ -120,6 +159,20 @@ func ImpersonateSelf(impersonationlevel uint32) (err error) { return } +func IsValidSid(sid *syscall.SID) (valid bool) { + r0, _, _ := syscall.Syscall(procIsValidSid.Addr(), 1, uintptr(unsafe.Pointer(sid)), 0, 0) + valid = r0 != 0 + return +} + +func LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32, token *syscall.Token) (err error) { + r1, _, e1 := syscall.Syscall6(procLogonUserW.Addr(), 6, uintptr(unsafe.Pointer(username)), uintptr(unsafe.Pointer(domain)), uintptr(unsafe.Pointer(password)), uintptr(logonType), uintptr(logonProvider), uintptr(unsafe.Pointer(token))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func LookupPrivilegeValue(systemname *uint16, name *uint16, luid *LUID) (err error) { r1, _, e1 := syscall.Syscall(procLookupPrivilegeValueW.Addr(), 3, uintptr(unsafe.Pointer(systemname)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid))) if r1 == 0 { @@ -266,6 +319,15 @@ func GetModuleFileName(module syscall.Handle, fn *uint16, len uint32) (n uint32, return } +func GetModuleHandle(modulename *uint16) (handle syscall.Handle, err error) { + r0, _, e1 := syscall.Syscall(procGetModuleHandleW.Addr(), 1, uintptr(unsafe.Pointer(modulename)), 0, 0) + handle = syscall.Handle(r0) + if handle == 0 { + err = errnoErr(e1) + } + return +} + func GetTempPath2(buflen uint32, buf *uint16) (n uint32, err error) { r0, _, e1 := syscall.Syscall(procGetTempPath2W.Addr(), 2, uintptr(buflen), uintptr(unsafe.Pointer(buf)), 0) n = uint32(r0) @@ -384,6 +446,22 @@ func NetShareDel(serverName *uint16, netName *uint16, reserved uint32) (neterr e return } +func NetUserAdd(serverName *uint16, level uint32, buf *byte, parmErr *uint32) (neterr error) { + r0, _, _ := syscall.Syscall6(procNetUserAdd.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(level), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(parmErr)), 0, 0) + if r0 != 0 { + neterr = syscall.Errno(r0) + } + return +} + +func NetUserDel(serverName *uint16, userName *uint16) (neterr error) { + r0, _, _ := syscall.Syscall(procNetUserDel.Addr(), 2, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(userName)), 0) + if r0 != 0 { + neterr = syscall.Errno(r0) + } + return +} + func NetUserGetLocalGroups(serverName *uint16, userName *uint16, level uint32, flags uint32, buf **byte, prefMaxLen uint32, entriesRead *uint32, totalEntries *uint32) (neterr error) { r0, _, _ := syscall.Syscall9(procNetUserGetLocalGroups.Addr(), 8, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(userName)), uintptr(level), uintptr(flags), uintptr(unsafe.Pointer(buf)), uintptr(prefMaxLen), uintptr(unsafe.Pointer(entriesRead)), uintptr(unsafe.Pointer(totalEntries)), 0) if r0 != 0 { @@ -392,11 +470,41 @@ func NetUserGetLocalGroups(serverName *uint16, userName *uint16, level uint32, f return } +func NtCreateFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, allocationSize *int64, attributes uint32, share uint32, disposition uint32, options uint32, eabuffer uintptr, ealength uint32) (ntstatus error) { + r0, _, _ := syscall.Syscall12(procNtCreateFile.Addr(), 11, uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(allocationSize)), uintptr(attributes), uintptr(share), uintptr(disposition), uintptr(options), uintptr(eabuffer), uintptr(ealength), 0) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + +func NtOpenFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, share uint32, options uint32) (ntstatus error) { + r0, _, _ := syscall.Syscall6(procNtOpenFile.Addr(), 6, uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(options)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + +func NtSetInformationFile(handle syscall.Handle, iosb *IO_STATUS_BLOCK, inBuffer uintptr, inBufferLen uint32, class uint32) (ntstatus error) { + r0, _, _ := syscall.Syscall6(procNtSetInformationFile.Addr(), 5, uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(inBuffer), uintptr(inBufferLen), uintptr(class), 0) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + func rtlGetVersion(info *_OSVERSIONINFOW) { syscall.Syscall(procRtlGetVersion.Addr(), 1, uintptr(unsafe.Pointer(info)), 0, 0) return } +func rtlNtStatusToDosErrorNoTeb(ntstatus NTStatus) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(ntstatus), 0, 0) + ret = syscall.Errno(r0) + return +} + func GetProcessMemoryInfo(handle syscall.Handle, memCounters *PROCESS_MEMORY_COUNTERS, cb uint32) (err error) { r1, _, e1 := syscall.Syscall(procGetProcessMemoryInfo.Addr(), 3, uintptr(handle), uintptr(unsafe.Pointer(memCounters)), uintptr(cb)) if r1 == 0 { diff --git a/src/go/build/syslist.go b/src/internal/syslist/syslist.go similarity index 75% rename from src/go/build/syslist.go rename to src/internal/syslist/syslist.go index 783bbe69..2349b6ea 100644 --- a/src/go/build/syslist.go +++ b/src/internal/syslist/syslist.go @@ -2,16 +2,19 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package build +// Package syslist stores tables of OS and ARCH names that are +// (or at one point were) acceptable build targets. + +package syslist // Note that this file is read by internal/goarch/gengoarch.go and by // internal/goos/gengoos.go. If you change this file, look at those // files as well. -// knownOS is the list of past, present, and future known GOOS values. +// KnownOS is the list of past, present, and future known GOOS values. // Do not remove from this list, as it is used for filename matching. -// If you add an entry to this list, look at unixOS, below. -var knownOS = map[string]bool{ +// If you add an entry to this list, look at UnixOS, below. +var KnownOS = map[string]bool{ "aix": true, "android": true, "darwin": true, @@ -32,11 +35,10 @@ var knownOS = map[string]bool{ "zos": true, } -// unixOS is the set of GOOS values matched by the "unix" build tag. +// UnixOS is the set of GOOS values matched by the "unix" build tag. // This is not used for filename matching. -// This list also appears in cmd/dist/build.go and -// cmd/go/internal/imports/build.go. -var unixOS = map[string]bool{ +// This list also appears in cmd/dist/build.go. +var UnixOS = map[string]bool{ "aix": true, "android": true, "darwin": true, @@ -51,9 +53,9 @@ var unixOS = map[string]bool{ "solaris": true, } -// knownArch is the list of past, present, and future known GOARCH values. +// KnownArch is the list of past, present, and future known GOARCH values. // Do not remove from this list, as it is used for filename matching. -var knownArch = map[string]bool{ +var KnownArch = map[string]bool{ "386": true, "amd64": true, "amd64p32": true, diff --git a/src/internal/testenv/exec.go b/src/internal/testenv/exec.go index 7f6ad5ca..7b251b60 100644 --- a/src/internal/testenv/exec.go +++ b/src/internal/testenv/exec.go @@ -31,20 +31,17 @@ import ( // If exec is not supported, testenv.SyscallIsNotSupported will return true // for the resulting error. func MustHaveExec(t testing.TB) { - tryExecOnce.Do(func() { - tryExecErr = tryExec() - }) - if tryExecErr != nil { - t.Skipf("skipping test: cannot exec subprocess on %s/%s: %v", runtime.GOOS, runtime.GOARCH, tryExecErr) + if err := tryExec(); err != nil { + msg := fmt.Sprintf("cannot exec subprocess on %s/%s: %v", runtime.GOOS, runtime.GOARCH, err) + if t == nil { + panic(msg) + } + t.Helper() + t.Skip("skipping test:", msg) } } -var ( - tryExecOnce sync.Once - tryExecErr error -) - -func tryExec() error { +var tryExec = sync.OnceValue(func() error { switch runtime.GOOS { case "wasip1", "js", "ios": default: @@ -70,15 +67,37 @@ func tryExec() error { // We know that this is a test executable. We should be able to run it with a // no-op flag to check for overall exec support. - exe, err := os.Executable() + exe, err := exePath() if err != nil { return fmt.Errorf("can't probe for exec support: %w", err) } cmd := exec.Command(exe, "-test.list=^$") cmd.Env = origEnv return cmd.Run() +}) + +// Executable is a wrapper around [MustHaveExec] and [os.Executable]. +// It returns the path name for the executable that started the current process, +// or skips the test if the current system can't start new processes, +// or fails the test if the path can not be obtained. +func Executable(t testing.TB) string { + MustHaveExec(t) + + exe, err := exePath() + if err != nil { + msg := fmt.Sprintf("os.Executable error: %v", err) + if t == nil { + panic(msg) + } + t.Fatal(msg) + } + return exe } +var exePath = sync.OnceValues(func() (string, error) { + return os.Executable() +}) + var execPaths sync.Map // path -> error // MustHaveExecPath checks that the current system can start the named executable @@ -93,6 +112,7 @@ func MustHaveExecPath(t testing.TB, path string) { err, _ = execPaths.LoadOrStore(path, err) } if err != nil { + t.Helper() t.Skipf("skipping test: %s: %s", path, err) } } diff --git a/src/internal/testenv/testenv.go b/src/internal/testenv/testenv.go index 9fb92406..9aecfaa6 100644 --- a/src/internal/testenv/testenv.go +++ b/src/internal/testenv/testenv.go @@ -53,63 +53,59 @@ func HasGoBuild() bool { return false } - goBuildOnce.Do(func() { - // To run 'go build', we need to be able to exec a 'go' command. - // We somewhat arbitrarily choose to exec 'go tool -n compile' because that - // also confirms that cmd/go can find the compiler. (Before CL 472096, - // we sometimes ended up with cmd/go installed in the test environment - // without a cmd/compile it could use to actually build things.) - cmd := exec.Command("go", "tool", "-n", "compile") - cmd.Env = origEnv - out, err := cmd.Output() - if err != nil { - goBuildErr = fmt.Errorf("%v: %w", cmd, err) - return - } - out = bytes.TrimSpace(out) - if len(out) == 0 { - goBuildErr = fmt.Errorf("%v: no tool reported", cmd) - return - } - if _, err := exec.LookPath(string(out)); err != nil { - goBuildErr = err - return - } - - if platform.MustLinkExternal(runtime.GOOS, runtime.GOARCH, false) { - // We can assume that we always have a complete Go toolchain available. - // However, this platform requires a C linker to build even pure Go - // programs, including tests. Do we have one in the test environment? - // (On Android, for example, the device running the test might not have a - // C toolchain installed.) - // - // If CC is set explicitly, assume that we do. Otherwise, use 'go env CC' - // to determine which toolchain it would use by default. - if os.Getenv("CC") == "" { - cmd := exec.Command("go", "env", "CC") - cmd.Env = origEnv - out, err := cmd.Output() - if err != nil { - goBuildErr = fmt.Errorf("%v: %w", cmd, err) - return - } - out = bytes.TrimSpace(out) - if len(out) == 0 { - goBuildErr = fmt.Errorf("%v: no CC reported", cmd) - return - } - _, goBuildErr = exec.LookPath(string(out)) - } - } - }) - - return goBuildErr == nil + return tryGoBuild() == nil } -var ( - goBuildOnce sync.Once - goBuildErr error -) +var tryGoBuild = sync.OnceValue(func() error { + // To run 'go build', we need to be able to exec a 'go' command. + // We somewhat arbitrarily choose to exec 'go tool -n compile' because that + // also confirms that cmd/go can find the compiler. (Before CL 472096, + // we sometimes ended up with cmd/go installed in the test environment + // without a cmd/compile it could use to actually build things.) + goTool, err := goTool() + if err != nil { + return err + } + cmd := exec.Command(goTool, "tool", "-n", "compile") + cmd.Env = origEnv + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("%v: %w", cmd, err) + } + out = bytes.TrimSpace(out) + if len(out) == 0 { + return fmt.Errorf("%v: no tool reported", cmd) + } + if _, err := exec.LookPath(string(out)); err != nil { + return err + } + + if platform.MustLinkExternal(runtime.GOOS, runtime.GOARCH, false) { + // We can assume that we always have a complete Go toolchain available. + // However, this platform requires a C linker to build even pure Go + // programs, including tests. Do we have one in the test environment? + // (On Android, for example, the device running the test might not have a + // C toolchain installed.) + // + // If CC is set explicitly, assume that we do. Otherwise, use 'go env CC' + // to determine which toolchain it would use by default. + if os.Getenv("CC") == "" { + cmd := exec.Command(goTool, "env", "CC") + cmd.Env = origEnv + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("%v: %w", cmd, err) + } + out = bytes.TrimSpace(out) + if len(out) == 0 { + return fmt.Errorf("%v: no CC reported", cmd) + } + _, err = exec.LookPath(string(out)) + return err + } + } + return nil +}) // MustHaveGoBuild checks that the current system can build programs with “go build” // and then run them with os.StartProcess or exec.Command. @@ -121,7 +117,7 @@ func MustHaveGoBuild(t testing.TB) { } if !HasGoBuild() { t.Helper() - t.Skipf("skipping test: 'go build' unavailable: %v", goBuildErr) + t.Skipf("skipping test: 'go build' unavailable: %v", tryGoBuild()) } } @@ -135,6 +131,7 @@ func HasGoRun() bool { // If not, MustHaveGoRun calls t.Skip with an explanation. func MustHaveGoRun(t testing.TB) { if !HasGoRun() { + t.Helper() t.Skipf("skipping test: 'go run' not available on %s/%s", runtime.GOOS, runtime.GOARCH) } } @@ -154,6 +151,7 @@ func HasParallelism() bool { // threads in parallel. If not, MustHaveParallelism calls t.Skip with an explanation. func MustHaveParallelism(t testing.TB) { if !HasParallelism() { + t.Helper() t.Skipf("skipping test: no parallelism available on %s/%s", runtime.GOOS, runtime.GOARCH) } } @@ -177,82 +175,67 @@ func GoToolPath(t testing.TB) string { return path } -var ( - gorootOnce sync.Once - gorootPath string - gorootErr error -) +var findGOROOT = sync.OnceValues(func() (path string, err error) { + if path := runtime.GOROOT(); path != "" { + // If runtime.GOROOT() is non-empty, assume that it is valid. + // + // (It might not be: for example, the user may have explicitly set GOROOT + // to the wrong directory. But this case is + // rare, and if that happens the user can fix what they broke.) + return path, nil + } -func findGOROOT() (string, error) { - gorootOnce.Do(func() { - gorootPath = runtime.GOROOT() - if gorootPath != "" { - // If runtime.GOROOT() is non-empty, assume that it is valid. - // - // (It might not be: for example, the user may have explicitly set GOROOT - // to the wrong directory. But this case is - // rare, and if that happens the user can fix what they broke.) - return + // runtime.GOROOT doesn't know where GOROOT is (perhaps because the test + // binary was built with -trimpath). + // + // Since this is internal/testenv, we can cheat and assume that the caller + // is a test of some package in a subdirectory of GOROOT/src. ('go test' + // runs the test in the directory containing the packaged under test.) That + // means that if we start walking up the tree, we should eventually find + // GOROOT/src/go.mod, and we can report the parent directory of that. + // + // Notably, this works even if we can't run 'go env GOROOT' as a + // subprocess. + + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("finding GOROOT: %w", err) + } + + dir := cwd + for { + parent := filepath.Dir(dir) + if parent == dir { + // dir is either "." or only a volume name. + return "", fmt.Errorf("failed to locate GOROOT/src in any parent directory") } - // runtime.GOROOT doesn't know where GOROOT is (perhaps because the test - // binary was built with -trimpath). - // - // Since this is internal/testenv, we can cheat and assume that the caller - // is a test of some package in a subdirectory of GOROOT/src. ('go test' - // runs the test in the directory containing the packaged under test.) That - // means that if we start walking up the tree, we should eventually find - // GOROOT/src/go.mod, and we can report the parent directory of that. - // - // Notably, this works even if we can't run 'go env GOROOT' as a - // subprocess. + if base := filepath.Base(dir); base != "src" { + dir = parent + continue // dir cannot be GOROOT/src if it doesn't end in "src". + } - cwd, err := os.Getwd() + b, err := os.ReadFile(filepath.Join(dir, "go.mod")) if err != nil { - gorootErr = fmt.Errorf("finding GOROOT: %w", err) - return - } - - dir := cwd - for { - parent := filepath.Dir(dir) - if parent == dir { - // dir is either "." or only a volume name. - gorootErr = fmt.Errorf("failed to locate GOROOT/src in any parent directory") - return - } - - if base := filepath.Base(dir); base != "src" { + if os.IsNotExist(err) { dir = parent - continue // dir cannot be GOROOT/src if it doesn't end in "src". + continue } + return "", fmt.Errorf("finding GOROOT: %w", err) + } + goMod := string(b) - b, err := os.ReadFile(filepath.Join(dir, "go.mod")) - if err != nil { - if os.IsNotExist(err) { - dir = parent - continue - } - gorootErr = fmt.Errorf("finding GOROOT: %w", err) - return - } - goMod := string(b) - - for goMod != "" { - var line string - line, goMod, _ = strings.Cut(goMod, "\n") - fields := strings.Fields(line) - if len(fields) >= 2 && fields[0] == "module" && fields[1] == "std" { - // Found "module std", which is the module declaration in GOROOT/src! - gorootPath = parent - return - } + for goMod != "" { + var line string + line, goMod, _ = strings.Cut(goMod, "\n") + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "module" && fields[1] == "std" { + // Found "module std", which is the module declaration in GOROOT/src! + return parent, nil } } - }) - - return gorootPath, gorootErr -} + } +}) // GOROOT reports the path to the directory containing the root of the Go // project source tree. This is normally equivalent to runtime.GOROOT, but @@ -278,25 +261,21 @@ func GoTool() (string, error) { if !HasGoBuild() { return "", errors.New("platform cannot run go tool") } - goToolOnce.Do(func() { - goToolPath, goToolErr = exec.LookPath("go") - }) - return goToolPath, goToolErr + return goTool() } -var ( - goToolOnce sync.Once - goToolPath string - goToolErr error -) +var goTool = sync.OnceValues(func() (string, error) { + return exec.LookPath("go") +}) -// HasSrc reports whether the entire source tree is available under GOROOT. -func HasSrc() bool { +// MustHaveSource checks that the entire source tree is available under GOROOT. +// If not, it calls t.Skip with an explanation. +func MustHaveSource(t testing.TB) { switch runtime.GOOS { case "ios": - return false + t.Helper() + t.Skip("skipping test: no source tree on " + runtime.GOOS) } - return true } // HasExternalNetwork reports whether the current system can use @@ -321,33 +300,31 @@ func MustHaveExternalNetwork(t testing.TB) { // HasCGO reports whether the current system can use cgo. func HasCGO() bool { - hasCgoOnce.Do(func() { - goTool, err := GoTool() - if err != nil { - return - } - cmd := exec.Command(goTool, "env", "CGO_ENABLED") - cmd.Env = origEnv - out, err := cmd.Output() - if err != nil { - panic(fmt.Sprintf("%v: %v", cmd, out)) - } - hasCgo, err = strconv.ParseBool(string(bytes.TrimSpace(out))) - if err != nil { - panic(fmt.Sprintf("%v: non-boolean output %q", cmd, out)) - } - }) - return hasCgo + return hasCgo() } -var ( - hasCgoOnce sync.Once - hasCgo bool -) +var hasCgo = sync.OnceValue(func() bool { + goTool, err := goTool() + if err != nil { + return false + } + cmd := exec.Command(goTool, "env", "CGO_ENABLED") + cmd.Env = origEnv + out, err := cmd.Output() + if err != nil { + panic(fmt.Sprintf("%v: %v", cmd, out)) + } + ok, err := strconv.ParseBool(string(bytes.TrimSpace(out))) + if err != nil { + panic(fmt.Sprintf("%v: non-boolean output %q", cmd, out)) + } + return ok +}) // MustHaveCGO calls t.Skip if cgo is not available. func MustHaveCGO(t testing.TB) { if !HasCGO() { + t.Helper() t.Skipf("skipping test: no cgo") } } @@ -363,6 +340,7 @@ func CanInternalLink(withCgo bool) bool { // If not, MustInternalLink calls t.Skip with an explanation. func MustInternalLink(t testing.TB, withCgo bool) { if !CanInternalLink(withCgo) { + t.Helper() if withCgo && CanInternalLink(false) { t.Skipf("skipping test: internal linking on %s/%s is not supported with cgo", runtime.GOOS, runtime.GOARCH) } @@ -375,6 +353,7 @@ func MustInternalLink(t testing.TB, withCgo bool) { // If not, MustInternalLinkPIE calls t.Skip with an explanation. func MustInternalLinkPIE(t testing.TB) { if !platform.InternalLinkPIESupported(runtime.GOOS, runtime.GOARCH) { + t.Helper() t.Skipf("skipping test: internal linking for buildmode=pie on %s/%s is not supported", runtime.GOOS, runtime.GOARCH) } } @@ -384,6 +363,7 @@ func MustInternalLinkPIE(t testing.TB) { // If not, MustHaveBuildMode calls t.Skip with an explanation. func MustHaveBuildMode(t testing.TB, buildmode string) { if !platform.BuildModeSupported(runtime.Compiler, buildmode, runtime.GOOS, runtime.GOARCH) { + t.Helper() t.Skipf("skipping test: build mode %s on %s/%s is not supported by the %s compiler", buildmode, runtime.GOOS, runtime.GOARCH, runtime.Compiler) } } @@ -399,6 +379,7 @@ func HasSymlink() bool { func MustHaveSymlink(t testing.TB) { ok, reason := hasSymlink() if !ok { + t.Helper() t.Skipf("skipping test: cannot make symlinks on %s/%s: %s", runtime.GOOS, runtime.GOARCH, reason) } } @@ -415,6 +396,7 @@ func HasLink() bool { // If not, MustHaveLink calls t.Skip with an explanation. func MustHaveLink(t testing.TB) { if !HasLink() { + t.Helper() t.Skipf("skipping test: hardlinks are not supported on %s/%s", runtime.GOOS, runtime.GOARCH) } } @@ -422,15 +404,15 @@ func MustHaveLink(t testing.TB) { var flaky = flag.Bool("flaky", false, "run known-flaky tests too") func SkipFlaky(t testing.TB, issue int) { - t.Helper() if !*flaky { + t.Helper() t.Skipf("skipping known flaky test without the -flaky flag; see golang.org/issue/%d", issue) } } func SkipFlakyNet(t testing.TB) { - t.Helper() if v, _ := strconv.ParseBool(os.Getenv("GO_BUILDER_FLAKY_NET")); v { + t.Helper() t.Skip("skipping test on builder known to have frequent network failures") } } diff --git a/src/internal/testenv/testenv_notwin.go b/src/internal/testenv/testenv_notwin.go index 30e159a6..9dddea94 100644 --- a/src/internal/testenv/testenv_notwin.go +++ b/src/internal/testenv/testenv_notwin.go @@ -11,9 +11,10 @@ import ( "os" "path/filepath" "runtime" + "sync" ) -func hasSymlink() (ok bool, reason string) { +var hasSymlink = sync.OnceValues(func() (ok bool, reason string) { switch runtime.GOOS { case "plan9": return false, "" @@ -43,4 +44,4 @@ func hasSymlink() (ok bool, reason string) { } return true, "" -} +}) diff --git a/src/internal/testenv/testenv_windows.go b/src/internal/testenv/testenv_windows.go index 4802b139..eed53cdf 100644 --- a/src/internal/testenv/testenv_windows.go +++ b/src/internal/testenv/testenv_windows.go @@ -5,16 +5,14 @@ package testenv import ( + "errors" "os" "path/filepath" "sync" "syscall" ) -var symlinkOnce sync.Once -var winSymlinkErr error - -func initWinHasSymlink() { +var hasSymlink = sync.OnceValues(func() (bool, string) { tmpdir, err := os.MkdirTemp("", "symtest") if err != nil { panic("failed to create temp directory: " + err.Error()) @@ -22,26 +20,13 @@ func initWinHasSymlink() { defer os.RemoveAll(tmpdir) err = os.Symlink("target", filepath.Join(tmpdir, "symlink")) - if err != nil { - err = err.(*os.LinkError).Err - switch err { - case syscall.EWINDOWS, syscall.ERROR_PRIVILEGE_NOT_HELD: - winSymlinkErr = err - } - } -} - -func hasSymlink() (ok bool, reason string) { - symlinkOnce.Do(initWinHasSymlink) - - switch winSymlinkErr { - case nil: + switch { + case err == nil: return true, "" - case syscall.EWINDOWS: + case errors.Is(err, syscall.EWINDOWS): return false, ": symlinks are not supported on your version of Windows" - case syscall.ERROR_PRIVILEGE_NOT_HELD: + case errors.Is(err, syscall.ERROR_PRIVILEGE_NOT_HELD): return false, ": you don't have enough privileges to create symlinks" } - return false, "" -} +}) diff --git a/src/internal/testlog/log.go b/src/internal/testlog/log.go index 3c5f780a..d8b9dcfa 100644 --- a/src/internal/testlog/log.go +++ b/src/internal/testlog/log.go @@ -21,20 +21,19 @@ type Interface interface { } // logger is the current logger Interface. -// We use an atomic.Value in case test startup +// We use an atomic.Pointer in case test startup // is racing with goroutines started during init. // That must not cause a race detector failure, // although it will still result in limited visibility // into exactly what those goroutines do. -var logger atomic.Value +var logger atomic.Pointer[Interface] // SetLogger sets the test logger implementation for the current process. // It must be called only once, at process startup. func SetLogger(impl Interface) { - if logger.Load() != nil { + if !logger.CompareAndSwap(nil, &impl) { panic("testlog: SetLogger must be called only once") } - logger.Store(&impl) } // Logger returns the current test logger implementation. @@ -44,7 +43,7 @@ func Logger() Interface { if impl == nil { return nil } - return *impl.(*Interface) + return *impl } // Getenv calls Logger().Getenv, if a logger has been set. diff --git a/src/internal/trace/base.go b/src/internal/trace/base.go index 4cbd3e64..4f4ce486 100644 --- a/src/internal/trace/base.go +++ b/src/internal/trace/base.go @@ -19,11 +19,7 @@ import ( // maxArgs is the maximum number of arguments for "plain" events, // i.e. anything that could reasonably be represented as a baseEvent. -// -// TODO(mknyszek): This is only 6 instead of 5 because GoStatusStack -// has 5 arguments and needs to smuggle in a 6th. Figure out a way to -// shrink this in the future. -const maxArgs = 6 +const maxArgs = 5 // timedEventArgs is an array that is able to hold the arguments for any // timed event. diff --git a/src/internal/trace/event.go b/src/internal/trace/event.go index a5d5637e..a5c5aec2 100644 --- a/src/internal/trace/event.go +++ b/src/internal/trace/event.go @@ -6,6 +6,7 @@ package trace import ( "fmt" + "iter" "math" "strings" "time" @@ -265,24 +266,25 @@ type Stack struct { } // Frames is an iterator over the frames in a Stack. -func (s Stack) Frames(yield func(f StackFrame) bool) bool { - if s.id == 0 { - return true - } - stk := s.table.stacks.mustGet(s.id) - for _, pc := range stk.pcs { - f := s.table.pcs[pc] - sf := StackFrame{ - PC: f.pc, - Func: s.table.strings.mustGet(f.funcID), - File: s.table.strings.mustGet(f.fileID), - Line: f.line, +func (s Stack) Frames() iter.Seq[StackFrame] { + return func(yield func(StackFrame) bool) { + if s.id == 0 { + return } - if !yield(sf) { - return false + stk := s.table.stacks.mustGet(s.id) + for _, pc := range stk.pcs { + f := s.table.pcs[pc] + sf := StackFrame{ + PC: f.pc, + Func: s.table.strings.mustGet(f.funcID), + File: s.table.strings.mustGet(f.fileID), + Line: f.line, + } + if !yield(sf) { + return + } } } - return true } // NoStack is a sentinel value that can be compared against any Stack value, indicating @@ -306,7 +308,7 @@ type StackFrame struct { Line uint64 } -// ExperimentalEvent presents a raw view of an experimental event's arguments and thier names. +// ExperimentalEvent presents a raw view of an experimental event's arguments and their names. type ExperimentalEvent struct { // Name is the name of the event. Name string @@ -647,8 +649,9 @@ func (e Event) StateTransition() StateTransition { s = goStateTransition(e.ctx.G, GoSyscall, GoRunnable) s.Stack = e.Stack() // This event references the resource the event happened on. case go122.EvGoStatus, go122.EvGoStatusStack: - // N.B. ordering.advance populates e.base.extra. - s = goStateTransition(GoID(e.base.args[0]), GoState(e.base.extra(version.Go122)[0]), go122GoStatus2GoState[e.base.args[2]]) + packedStatus := e.base.args[2] + from, to := packedStatus>>32, packedStatus&((1<<32)-1) + s = goStateTransition(GoID(e.base.args[0]), GoState(from), go122GoStatus2GoState[to]) default: panic(fmt.Sprintf("internal error: unexpected event type for StateTransition kind: %s", go122.EventString(e.base.typ))) } @@ -795,11 +798,10 @@ func (e Event) String() string { if s.Stack != NoStack { fmt.Fprintln(&sb) fmt.Fprintln(&sb, "TransitionStack=") - s.Stack.Frames(func(f StackFrame) bool { + for f := range s.Stack.Frames() { fmt.Fprintf(&sb, "\t%s @ 0x%x\n", f.Func, f.PC) fmt.Fprintf(&sb, "\t\t%s:%d\n", f.File, f.Line) - return true - }) + } } case EventExperimental: r := e.Experimental() @@ -808,11 +810,10 @@ func (e Event) String() string { if stk := e.Stack(); stk != NoStack { fmt.Fprintln(&sb) fmt.Fprintln(&sb, "Stack=") - stk.Frames(func(f StackFrame) bool { + for f := range stk.Frames() { fmt.Fprintf(&sb, "\t%s @ 0x%x\n", f.Func, f.PC) fmt.Fprintf(&sb, "\t\t%s:%d\n", f.File, f.Line) - return true - }) + } } return sb.String() } diff --git a/src/internal/trace/event/event.go b/src/internal/trace/event/event.go index adcb8811..b8b6af00 100644 --- a/src/internal/trace/event/event.go +++ b/src/internal/trace/event/event.go @@ -4,7 +4,9 @@ package event -// Type is the common in-memory representation of the low-leve +// Type indicates an event's type from which its arguments and semantics can be +// derived. Its representation matches the wire format's representation of the event +// types that precede all event data. type Type uint8 // Spec is a specification for a trace event. It contains sufficient information @@ -18,7 +20,7 @@ type Spec struct { // // Argument names follow a certain structure and this structure // is relied on by the testing framework to type-check arguments. - // The structure is is: + // The structure is: // // (?P[A-Za-z]+_)?(?P[A-Za-z]+) // diff --git a/src/internal/trace/generation.go b/src/internal/trace/generation.go index 098d1d4f..98bbf439 100644 --- a/src/internal/trace/generation.go +++ b/src/internal/trace/generation.go @@ -25,6 +25,7 @@ import ( type generation struct { gen uint64 batches map[ThreadID][]batch + batchMs []ThreadID cpuSamples []cpuSample *evTable } @@ -169,6 +170,9 @@ func processBatch(g *generation, b batch) error { return err } default: + if _, ok := g.batches[b.m]; !ok { + g.batchMs = append(g.batchMs, b.m) + } g.batches[b.m] = append(g.batches[b.m], b) } return nil diff --git a/src/internal/trace/internal/oldtrace/parser.go b/src/internal/trace/internal/oldtrace/parser.go index afbf0ed5..0365eeff 100644 --- a/src/internal/trace/internal/oldtrace/parser.go +++ b/src/internal/trace/internal/oldtrace/parser.go @@ -385,10 +385,10 @@ func (p *parser) parseEventBatches() (Events, error) { // Merge events as long as at least one P has more events gs := make(map[uint64]gState) // Note: technically we don't need a priority queue here. We're only ever - // interested in the earliest elligible event, which means we just have to + // interested in the earliest eligible event, which means we just have to // track the smallest element. However, in practice, the priority queue // performs better, because for each event we only have to compute its state - // transition once, not on each iteration. If it was elligible before, it'll + // transition once, not on each iteration. If it was eligible before, it'll // already be in the queue. Furthermore, on average, we only have one P to // look at in each iteration, because all other Ps are already in the queue. var frontier orderEventList @@ -813,7 +813,7 @@ func (p *parser) readRawEvent(flags uint, ev *rawEvent) error { } } -// loadBatch loads the next batch for pid and appends its contents to to events. +// loadBatch loads the next batch for pid and appends its contents to events. func (p *parser) loadBatch(pid int32, events []Event) ([]Event, error) { offsets := p.batchOffsets[pid] if len(offsets) == 0 { diff --git a/src/internal/trace/order.go b/src/internal/trace/order.go index 4b3b8029..d0818a50 100644 --- a/src/internal/trace/order.go +++ b/src/internal/trace/order.go @@ -377,7 +377,7 @@ func (o *ordering) advanceGoStatus(ev *baseEvent, evt *evTable, m ThreadID, gen } else { return curCtx, false, fmt.Errorf("found goroutine status for new goroutine after the first generation: id=%v status=%v", gid, status) } - ev.extra(version.Go122)[0] = uint64(oldState) // Smuggle in the old state for StateTransition. + ev.args[2] = uint64(oldState)<<32 | uint64(status) // Smuggle in the old state for StateTransition. newCtx := curCtx switch status { diff --git a/src/internal/trace/raw/event.go b/src/internal/trace/raw/event.go index 4766fbe5..e163a2c6 100644 --- a/src/internal/trace/raw/event.go +++ b/src/internal/trace/raw/event.go @@ -5,6 +5,7 @@ package raw import ( + "encoding/binary" "strconv" "strings" @@ -58,3 +59,18 @@ func (e *Event) String() string { } return s.String() } + +// EncodedSize returns the canonical encoded size of an event. +func (e *Event) EncodedSize() int { + size := 1 + var buf [binary.MaxVarintLen64]byte + for _, arg := range e.Args { + size += binary.PutUvarint(buf[:], arg) + } + spec := e.Version.Specs()[e.Ev] + if spec.HasData { + size += binary.PutUvarint(buf[:], uint64(len(e.Data))) + size += len(e.Data) + } + return size +} diff --git a/src/internal/trace/reader.go b/src/internal/trace/reader.go index c05d5b58..81157292 100644 --- a/src/internal/trace/reader.go +++ b/src/internal/trace/reader.go @@ -142,7 +142,8 @@ func (r *Reader) ReadEvent() (e Event, err error) { r.cpuSamples = r.gen.cpuSamples // Reset frontier. - for m, batches := range r.gen.batches { + for _, m := range r.gen.batchMs { + batches := r.gen.batches[m] bc := &batchCursor{m: m} ok, err := bc.nextEvent(batches, r.gen.freq) if err != nil { diff --git a/src/internal/trace/summary.go b/src/internal/trace/summary.go index fa3e3359..f31439fe 100644 --- a/src/internal/trace/summary.go +++ b/src/internal/trace/summary.go @@ -390,24 +390,14 @@ func (s *Summarizer) Event(ev *Event) { // This root frame will be identical for all transitions on this // goroutine, because it represents its immutable start point. if g.Name == "" { - stk := st.Stack - if stk != NoStack { - var frame StackFrame - var ok bool - stk.Frames(func(f StackFrame) bool { - frame = f - ok = true - return true - }) - if ok { - // NB: this PC won't actually be consistent for - // goroutines which existed at the start of the - // trace. The UI doesn't use it directly; this - // mainly serves as an indication that we - // actually saw a call stack for the goroutine - g.PC = frame.PC - g.Name = frame.Func - } + for frame := range st.Stack.Frames() { + // NB: this PC won't actually be consistent for + // goroutines which existed at the start of the + // trace. The UI doesn't use it directly; this + // mainly serves as an indication that we + // actually saw a call stack for the goroutine + g.PC = frame.PC + g.Name = frame.Func } } diff --git a/src/internal/trace/testdata/cmd/gotraceeventstats/main.go b/src/internal/trace/testdata/cmd/gotraceeventstats/main.go deleted file mode 100644 index d2379daa..00000000 --- a/src/internal/trace/testdata/cmd/gotraceeventstats/main.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "cmp" - "encoding/binary" - "flag" - "fmt" - "io" - "log" - "os" - "slices" - "text/tabwriter" - - "internal/trace/event" - "internal/trace/raw" -) - -func init() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [mode]\n", os.Args[0]) - fmt.Fprintf(flag.CommandLine.Output(), "\n") - fmt.Fprintf(flag.CommandLine.Output(), "Accepts a trace at stdin.\n") - fmt.Fprintf(flag.CommandLine.Output(), "\n") - fmt.Fprintf(flag.CommandLine.Output(), "Supported modes:") - fmt.Fprintf(flag.CommandLine.Output(), "\n") - fmt.Fprintf(flag.CommandLine.Output(), "* size - dumps size stats\n") - fmt.Fprintf(flag.CommandLine.Output(), "\n") - flag.PrintDefaults() - } - log.SetFlags(0) -} - -func main() { - log.SetPrefix("") - flag.Parse() - - if flag.NArg() != 1 { - log.Print("missing mode argument") - flag.Usage() - os.Exit(1) - } - var err error - switch mode := flag.Arg(0); mode { - case "size": - err = printSizeStats(os.Stdin) - default: - log.Printf("unknown mode %q", mode) - flag.Usage() - os.Exit(1) - } - if err != nil { - log.Fatalf("error: %v", err) - os.Exit(1) - } -} - -func printSizeStats(r io.Reader) error { - cr := countingReader{Reader: r} - tr, err := raw.NewReader(&cr) - if err != nil { - return err - } - type eventStats struct { - typ event.Type - count int - bytes int - } - var stats [256]eventStats - for i := range stats { - stats[i].typ = event.Type(i) - } - eventsRead := 0 - for { - e, err := tr.ReadEvent() - if err == io.EOF { - break - } - if err != nil { - return err - } - s := &stats[e.Ev] - s.count++ - s.bytes += encodedSize(&e) - eventsRead++ - } - slices.SortFunc(stats[:], func(a, b eventStats) int { - return cmp.Compare(b.bytes, a.bytes) - }) - specs := tr.Version().Specs() - w := tabwriter.NewWriter(os.Stdout, 3, 8, 2, ' ', 0) - fmt.Fprintf(w, "Event\tBytes\t%%\tCount\t%%\n") - fmt.Fprintf(w, "-\t-\t-\t-\t-\n") - for i := range stats { - stat := &stats[i] - name := "" - if int(stat.typ) >= len(specs) { - name = fmt.Sprintf("", stat.typ) - } else { - name = specs[stat.typ].Name - } - bytesPct := float64(stat.bytes) / float64(cr.bytesRead) * 100 - countPct := float64(stat.count) / float64(eventsRead) * 100 - fmt.Fprintf(w, "%s\t%d\t%.2f%%\t%d\t%.2f%%\n", name, stat.bytes, bytesPct, stat.count, countPct) - } - w.Flush() - return nil -} - -func encodedSize(e *raw.Event) int { - size := 1 - var buf [binary.MaxVarintLen64]byte - for _, arg := range e.Args { - size += binary.PutUvarint(buf[:], arg) - } - spec := e.Version.Specs()[e.Ev] - if spec.HasData { - size += binary.PutUvarint(buf[:], uint64(len(e.Data))) - size += len(e.Data) - } - return size -} - -type countingReader struct { - io.Reader - bytesRead int -} - -func (r *countingReader) Read(b []byte) (int, error) { - n, err := r.Reader.Read(b) - r.bytesRead += n - return n, err -} diff --git a/src/internal/trace/testdata/cmd/gotraceraw/main.go b/src/internal/trace/testdata/cmd/gotraceraw/main.go deleted file mode 100644 index ec4ebf23..00000000 --- a/src/internal/trace/testdata/cmd/gotraceraw/main.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "io" - "log" - "os" - - "internal/trace/raw" - "internal/trace/version" -) - -func init() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [mode]\n", os.Args[0]) - fmt.Fprintf(flag.CommandLine.Output(), "\n") - fmt.Fprintf(flag.CommandLine.Output(), "Supported modes:") - fmt.Fprintf(flag.CommandLine.Output(), "\n") - fmt.Fprintf(flag.CommandLine.Output(), "* text2bytes - converts a text format trace to bytes\n") - fmt.Fprintf(flag.CommandLine.Output(), "* bytes2text - converts a byte format trace to text\n") - fmt.Fprintf(flag.CommandLine.Output(), "\n") - flag.PrintDefaults() - } - log.SetFlags(0) -} - -func main() { - flag.Parse() - if narg := flag.NArg(); narg != 1 { - log.Fatal("expected exactly one positional argument: the mode to operate in; see -h output") - } - - r := os.Stdin - w := os.Stdout - - var tr traceReader - var tw traceWriter - var err error - - switch flag.Arg(0) { - case "text2bytes": - tr, err = raw.NewTextReader(r) - if err != nil { - log.Fatal(err) - } - tw, err = raw.NewWriter(w, tr.Version()) - if err != nil { - log.Fatal(err) - } - case "bytes2text": - tr, err = raw.NewReader(r) - if err != nil { - log.Fatal(err) - } - tw, err = raw.NewTextWriter(w, tr.Version()) - if err != nil { - log.Fatal(err) - } - } - for { - ev, err := tr.ReadEvent() - if err == io.EOF { - break - } - if err != nil { - log.Fatal(err) - } - if err := tw.WriteEvent(ev); err != nil { - log.Fatal(err) - } - } -} - -type traceReader interface { - Version() version.Version - ReadEvent() (raw.Event, error) -} - -type traceWriter interface { - WriteEvent(raw.Event) error -} diff --git a/src/internal/trace/testdata/cmd/gotracevalidate/main.go b/src/internal/trace/testdata/cmd/gotracevalidate/main.go deleted file mode 100644 index 6c681df1..00000000 --- a/src/internal/trace/testdata/cmd/gotracevalidate/main.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "io" - "log" - "os" - - "internal/trace" - "internal/trace/testtrace" -) - -func init() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s\n", os.Args[0]) - fmt.Fprintf(flag.CommandLine.Output(), "\n") - fmt.Fprintf(flag.CommandLine.Output(), "Accepts a trace at stdin and validates it.\n") - flag.PrintDefaults() - } - log.SetFlags(0) -} - -var logEvents = flag.Bool("log-events", false, "whether to log events") - -func main() { - flag.Parse() - - r, err := trace.NewReader(os.Stdin) - if err != nil { - log.Fatal(err) - } - v := testtrace.NewValidator() - for { - ev, err := r.ReadEvent() - if err == io.EOF { - break - } - if err != nil { - log.Fatal(err) - } - if *logEvents { - log.Println(ev.String()) - } - if err := v.Event(ev); err != nil { - log.Fatal(err) - } - } -} diff --git a/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go b/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go index a14d3767..e5081598 100644 --- a/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go +++ b/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go @@ -4,7 +4,7 @@ // Tests a G being created from within a syscall. // -// Specifically, it tests a scenerio wherein a C +// Specifically, it tests a scenario wherein a C // thread is calling into Go, creating a goroutine in // a syscall (in the tracer's model). The system is free // to reuse thread IDs, so first a thread ID is used to diff --git a/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go b/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go index 1eb18026..681464ce 100644 --- a/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go +++ b/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go @@ -4,7 +4,7 @@ // Tests a G being created from within a syscall. // -// Specifically, it tests a scenerio wherein a C +// Specifically, it tests a scenario wherein a C // thread is calling into Go, creating a goroutine in // a syscall (in the tracer's model). Because the actual // m can be reused, it's possible for that m to have never diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go index 1f01cc9e..4e729698 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go @@ -4,7 +4,7 @@ // Tests syscall P stealing. // -// Specifically, it tests a scenerio wherein, without a +// Specifically, it tests a scenario wherein, without a // P sequence number of GoSyscallBegin, the syscall that // a ProcSteal applies to is ambiguous. This only happens in // practice when the events aren't already properly ordered diff --git a/src/internal/trace/testtrace/validation.go b/src/internal/trace/testtrace/validation.go index ec492110..59ff19e6 100644 --- a/src/internal/trace/testtrace/validation.go +++ b/src/internal/trace/testtrace/validation.go @@ -350,20 +350,17 @@ func (v *Validator) getOrCreateThread(e *errAccumulator, ev trace.Event, m trace func checkStack(e *errAccumulator, stk trace.Stack) { // Check for non-empty values, but we also check for crashes due to incorrect validation. - i := 0 - stk.Frames(func(f trace.StackFrame) bool { + for i, f := range slices.Collect(stk.Frames()) { if i == 0 { // Allow for one fully zero stack. // // TODO(mknyszek): Investigate why that happens. - return true + continue } if f.Func == "" || f.File == "" || f.PC == 0 || f.Line == 0 { e.Errorf("invalid stack frame %#v: missing information", f) } - i++ - return true - }) + } } type errAccumulator struct { diff --git a/src/internal/trace/trace_test.go b/src/internal/trace/trace_test.go index 7dc5cbb8..facac47e 100644 --- a/src/internal/trace/trace_test.go +++ b/src/internal/trace/trace_test.go @@ -16,6 +16,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" "testing" ) @@ -148,12 +149,11 @@ func TestTraceCPUProfile(t *testing.T) { if hogRegion != nil && ev.Goroutine() == hogRegion.Goroutine() { traceSamples++ var fns []string - ev.Stack().Frames(func(frame trace.StackFrame) bool { + for frame := range ev.Stack().Frames() { if frame.Func != "runtime.goexit" { fns = append(fns, fmt.Sprintf("%s:%d", frame.Func, frame.Line)) } - return true - }) + } stack := strings.Join(fns, "|") traceStacks[stack]++ } @@ -436,21 +436,15 @@ func TestTraceStacks(t *testing.T) { }...) } stackMatches := func(stk trace.Stack, frames []frame) bool { - i := 0 - match := true - stk.Frames(func(f trace.StackFrame) bool { + for i, f := range slices.Collect(stk.Frames()) { if f.Func != frames[i].fn { - match = false return false } if line := uint64(frames[i].line); line != 0 && line != f.Line { - match = false return false } - i++ - return true - }) - return match + } + return true } r, err := trace.NewReader(bytes.NewReader(tb)) if err != nil { @@ -507,7 +501,7 @@ func TestTraceStress(t *testing.T) { case "js", "wasip1": t.Skip("no os.Pipe on " + runtime.GOOS) } - testTraceProg(t, "stress.go", nil) + testTraceProg(t, "stress.go", checkReaderDeterminism) } func TestTraceStressStartStop(t *testing.T) { @@ -535,6 +529,43 @@ func TestTraceIterPull(t *testing.T) { testTraceProg(t, "iter-pull.go", nil) } +func checkReaderDeterminism(t *testing.T, tb, _ []byte, _ bool) { + events := func() []trace.Event { + var evs []trace.Event + + r, err := trace.NewReader(bytes.NewReader(tb)) + if err != nil { + t.Error(err) + } + for { + ev, err := r.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + evs = append(evs, ev) + } + + return evs + } + + evs1 := events() + evs2 := events() + + if l1, l2 := len(evs1), len(evs2); l1 != l2 { + t.Fatalf("re-reading trace gives different event count (%d != %d)", l1, l2) + } + for i, ev1 := range evs1 { + ev2 := evs2[i] + if s1, s2 := ev1.String(), ev2.String(); s1 != s2 { + t.Errorf("re-reading trace gives different event %d:\n%s\n%s\n", i, s1, s2) + break + } + } +} + func testTraceProg(t *testing.T, progName string, extra func(t *testing.T, trace, stderr []byte, stress bool)) { testenv.MustHaveGoRun(t) diff --git a/src/internal/trace/traceviewer/emitter.go b/src/internal/trace/traceviewer/emitter.go index c91c743a..c74f1c2e 100644 --- a/src/internal/trace/traceviewer/emitter.go +++ b/src/internal/trace/traceviewer/emitter.go @@ -614,7 +614,7 @@ func (e *Emitter) tsWithinRange(ts time.Duration) bool { return e.rangeStart <= ts && ts <= e.rangeEnd } -// OptionalEvent emits ev if it's within the time range of of the consumer, i.e. +// OptionalEvent emits ev if it's within the time range of the consumer, i.e. // the selected trace split range. func (e *Emitter) OptionalEvent(ev *format.Event) { e.c.ConsumeViewerEvent(ev, false) diff --git a/src/internal/types/errors/generrordocs.go b/src/internal/types/errors/generrordocs.go index 46343be3..613c7742 100644 --- a/src/internal/types/errors/generrordocs.go +++ b/src/internal/types/errors/generrordocs.go @@ -1,9 +1,9 @@ -//go:build ignore - // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build ignore + // generrordocs creates a Markdown file for each (compiler) error code // and its associated documentation. // Note: this program must be run in this directory. diff --git a/src/internal/types/testdata/check/builtins0.go b/src/internal/types/testdata/check/builtins0.go index 1c0e6920..62759d1e 100644 --- a/src/internal/types/testdata/check/builtins0.go +++ b/src/internal/types/testdata/check/builtins0.go @@ -515,7 +515,7 @@ func max1() { _ = max(s) _ = max(x, x) _ = max(x, x, x, x, x) - var _ int = max /* ERROR "cannot use max(m) (value of type myint) as int value" */ (m) + var _ int = max /* ERROR "cannot use max(m) (value of int type myint) as int value" */ (m) _ = max(x, m /* ERROR "invalid argument: mismatched types int (previous argument) and myint (type of m)" */ , x) _ = max(1, x) @@ -569,7 +569,7 @@ func min1() { _ = min(s) _ = min(x, x) _ = min(x, x, x, x, x) - var _ int = min /* ERROR "cannot use min(m) (value of type myint) as int value" */ (m) + var _ int = min /* ERROR "cannot use min(m) (value of int type myint) as int value" */ (m) _ = min(x, m /* ERROR "invalid argument: mismatched types int (previous argument) and myint (type of m)" */ , x) _ = min(1, x) diff --git a/src/internal/types/testdata/check/decls2/decls2a.go b/src/internal/types/testdata/check/decls2/decls2a.go index 2362bb96..58fdbfe1 100644 --- a/src/internal/types/testdata/check/decls2/decls2a.go +++ b/src/internal/types/testdata/check/decls2/decls2a.go @@ -83,7 +83,7 @@ func (T5 /* ERROR "invalid receiver" */ ) m2() {} // Methods associated with a named pointer type. type ptr *int func (ptr /* ERROR "invalid receiver" */ ) _() {} -func (* /* ERROR "invalid receiver" */ ptr) _() {} +func (*ptr /* ERROR "invalid receiver" */ ) _() {} // Methods with zero or multiple receivers. func ( /* ERROR "method has no receiver" */ ) _() {} @@ -96,13 +96,13 @@ func (a, b, c /* ERROR "method has multiple receivers" */ T3) _() {} func (int /* ERROR "cannot define new methods on non-local type int" */ ) m() {} func ([ /* ERROR "invalid receiver" */ ]int) m() {} func (time /* ERROR "cannot define new methods on non-local type time.Time" */ .Time) m() {} -func (* /* ERROR "cannot define new methods on non-local type time.Time" */ time.Time) m() {} -func (x /* ERROR "invalid receiver" */ interface{}) m() {} +func (*time /* ERROR "cannot define new methods on non-local type time.Time" */ .Time) m() {} +func (x any /* ERROR "invalid receiver" */ ) m() {} // Unsafe.Pointer is treated like a pointer when used as receiver type. type UP unsafe.Pointer func (UP /* ERROR "invalid" */ ) m1() {} -func (* /* ERROR "invalid" */ UP) m2() {} +func (*UP /* ERROR "invalid" */ ) m2() {} // Double declarations across package files const c_double = 0 diff --git a/src/internal/types/testdata/check/errors.go b/src/internal/types/testdata/check/errors.go index 10b6a22e..615cf862 100644 --- a/src/internal/types/testdata/check/errors.go +++ b/src/internal/types/testdata/check/errors.go @@ -58,7 +58,7 @@ func _() { // Use unqualified names for package-local objects. type T struct{} -var _ int = T /* ERROR "value of type T" */ {} // use T in error message rather than errors.T +var _ int = T /* ERROR "value of struct type T" */ {} // use T in error message rather than errors.T // Don't report errors containing "invalid type" (issue #24182). func _(x *missing /* ERROR "undefined: missing" */ ) { diff --git a/src/internal/types/testdata/check/go1_20_19.go b/src/internal/types/testdata/check/go1_20_19.go index e040d396..892179c7 100644 --- a/src/internal/types/testdata/check/go1_20_19.go +++ b/src/internal/types/testdata/check/go1_20_19.go @@ -14,4 +14,4 @@ type Slice []byte type Array [8]byte var s Slice -var p = (Array)(s /* ok because file versions below go1.21 set the langage version to go1.21 */) +var p = (Array)(s /* ok because file versions below go1.21 set the language version to go1.21 */) diff --git a/src/internal/types/testdata/check/go1_21_19.go b/src/internal/types/testdata/check/go1_21_19.go index 5866033e..febf653c 100644 --- a/src/internal/types/testdata/check/go1_21_19.go +++ b/src/internal/types/testdata/check/go1_21_19.go @@ -14,4 +14,4 @@ type Slice []byte type Array [8]byte var s Slice -var p = (Array)(s /* ok because file versions below go1.21 set the langage version to go1.21 */) +var p = (Array)(s /* ok because file versions below go1.21 set the language version to go1.21 */) diff --git a/src/internal/types/testdata/check/issues0.go b/src/internal/types/testdata/check/issues0.go index 3bf4a314..44a709d6 100644 --- a/src/internal/types/testdata/check/issues0.go +++ b/src/internal/types/testdata/check/issues0.go @@ -327,7 +327,7 @@ func issue28281c(a, b, c ... /* ERROR "can only use ... with final parameter" */ func issue28281d(... /* ERROR "can only use ... with final parameter" */ int, int) func issue28281e(a, b, c ... /* ERROR "can only use ... with final parameter" */ int, d int) func issue28281f(... /* ERROR "can only use ... with final parameter" */ int, ... /* ERROR "can only use ... with final parameter" */ int, int) -func (... /* ERROR "can only use ... with final parameter" */ TT) f() +func (... /* ERROR "invalid use of '...'" */ TT) f() func issue28281g() (... /* ERROR "can only use ... with final parameter" */ TT) // Issue #26234: Make various field/method lookup errors easier to read by matching cmd/compile's output @@ -363,7 +363,7 @@ func issue35895() { // Because both t1 and t2 have the same global package name (template), // qualify packages with full path name in this case. - var _ t1.Template = t2 /* ERRORx `cannot use .* \(value of type .html/template.\.Template\) as .text/template.\.Template` */ .Template{} + var _ t1.Template = t2 /* ERRORx `cannot use .* \(value of struct type .html/template.\.Template\) as .text/template.\.Template` */ .Template{} } func issue42989(s uint) { diff --git a/src/internal/types/testdata/check/typeparams.go b/src/internal/types/testdata/check/typeparams.go index b002377d..5fd82a5a 100644 --- a/src/internal/types/testdata/check/typeparams.go +++ b/src/internal/types/testdata/check/typeparams.go @@ -58,10 +58,10 @@ func min[T interface{ ~int }](x, y T) T { } func _[T interface{~int | ~float32}](x, y T) bool { return x < y } -func _[T any](x, y T) bool { return x /* ERROR "type parameter T is not comparable" */ < y } -func _[T interface{~int | ~float32 | ~bool}](x, y T) bool { return x /* ERROR "type parameter T is not comparable" */ < y } +func _[T any](x, y T) bool { return x /* ERROR "type parameter T cannot use operator <" */ < y } +func _[T interface{~int | ~float32 | ~bool}](x, y T) bool { return x /* ERROR "type parameter T cannot use operator <" */ < y } -func _[T C1[T]](x, y T) bool { return x /* ERROR "type parameter T is not comparable" */ < y } +func _[T C1[T]](x, y T) bool { return x /* ERROR "type parameter T cannot use operator <" */ < y } func _[T C2[T]](x, y T) bool { return x < y } type C1[T any] interface{} diff --git a/src/internal/types/testdata/fixedbugs/issue47818.go b/src/internal/types/testdata/fixedbugs/issue47818.go index 21c85392..4750a4fd 100644 --- a/src/internal/types/testdata/fixedbugs/issue47818.go +++ b/src/internal/types/testdata/fixedbugs/issue47818.go @@ -25,7 +25,7 @@ func f[P /* ERROR "type parameter requires go1.18 or later" */ any /* ERROR "pre _ = T[ /* ERROR "type instantiation requires go1.18 or later" */ int](struct{}{}) } -func (T[ /* ERROR "type instantiation requires go1.18 or later" */ P]) g(x int) { +func (T /* ERROR "type instantiation requires go1.18 or later" */ [P]) g(x int) { f[ /* ERROR "function instantiation requires go1.18 or later" */ int](0) // explicit instantiation (f[ /* ERROR "function instantiation requires go1.18 or later" */ int])(0) // parentheses (different code path) f( /* ERROR "implicit function instantiation requires go1.18 or later" */ x) // implicit instantiation diff --git a/src/internal/types/testdata/fixedbugs/issue47968.go b/src/internal/types/testdata/fixedbugs/issue47968.go index 83a17861..e260c63a 100644 --- a/src/internal/types/testdata/fixedbugs/issue47968.go +++ b/src/internal/types/testdata/fixedbugs/issue47968.go @@ -1,3 +1,5 @@ +// -gotypesalias=1 + // Copyright 2021 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -10,12 +12,12 @@ func (T[P]) m1() type A1 = T // ERROR "cannot use generic type" -func (A1[P]) m2() {} +func (A1[P]) m2() {} // don't report a follow-on error on A1 type A2 = T[int] -func (A2 /* ERRORx `cannot define new methods on instantiated type (T\[int\]|A2)` */) m3() {} -func (_ /* ERRORx `cannot define new methods on instantiated type (T\[int\]|A2)` */ A2) m4() {} +func (A2 /* ERROR "cannot define new methods on instantiated type T[int]" */) m3() {} +func (_ A2 /* ERROR "cannot define new methods on instantiated type T[int]" */) m4() {} -func (T[int]) m5() {} // int is the type parameter name, not an instantiation +func (T[int]) m5() {} // int is the type parameter name, not an instantiation func (T[* /* ERROR "must be an identifier" */ int]) m6() {} // syntax error diff --git a/src/internal/types/testdata/fixedbugs/issue48712.go b/src/internal/types/testdata/fixedbugs/issue48712.go index 76ad16cd..028660fb 100644 --- a/src/internal/types/testdata/fixedbugs/issue48712.go +++ b/src/internal/types/testdata/fixedbugs/issue48712.go @@ -10,7 +10,7 @@ func _[P comparable](x, y P) { _ = y == x _ = y == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } func _[P comparable](x P, y any) { @@ -19,7 +19,7 @@ func _[P comparable](x P, y any) { _ = y == x _ = y == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } func _[P any](x, y P) { @@ -28,7 +28,7 @@ func _[P any](x, y P) { _ = y /* ERROR "incomparable types in type set" */ == x _ = y /* ERROR "incomparable types in type set" */ == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } func _[P any](x P, y any) { @@ -37,5 +37,5 @@ func _[P any](x P, y any) { _ = y == x // ERROR "incomparable types in type set" _ = y == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } diff --git a/src/internal/types/testdata/fixedbugs/issue49005.go b/src/internal/types/testdata/fixedbugs/issue49005.go index d91c2078..6ec926ec 100644 --- a/src/internal/types/testdata/fixedbugs/issue49005.go +++ b/src/internal/types/testdata/fixedbugs/issue49005.go @@ -26,6 +26,6 @@ type X2 struct{} func _() { switch F2().(type) { - case * /* ERROR "impossible type switch case: *X2\n\tF2() (value of type T2) cannot have dynamic type *X2 (missing method M)" */ X2: + case * /* ERROR "impossible type switch case: *X2\n\tF2() (value of interface type T2) cannot have dynamic type *X2 (missing method M)" */ X2: } } diff --git a/src/internal/types/testdata/fixedbugs/issue51339.go b/src/internal/types/testdata/fixedbugs/issue51339.go index fd10daa2..933e4267 100644 --- a/src/internal/types/testdata/fixedbugs/issue51339.go +++ b/src/internal/types/testdata/fixedbugs/issue51339.go @@ -11,10 +11,10 @@ type T[P any, B *P] struct{} func (T /* ERROR "cannot use generic type" */) m0() {} -// TODO(rfindley): eliminate the duplicate errors here. -func ( /* ERROR "got 1 type parameter, but receiver base type declares 2" */ T /* ERROR "not enough type arguments for type" */ [_]) m1() { +func (T /* ERROR "receiver declares 1 type parameter, but receiver base type declares 2" */ [_]) m1() { } func (T[_, _]) m2() {} // TODO(gri) this error is unfortunate (issue #51343) -func (T /* ERROR "too many type arguments for type" */ [_, _, _]) m3() {} +func (T /* ERROR "receiver declares 3 type parameters, but receiver base type declares 2" */ [_, _, _]) m3() { +} diff --git a/src/internal/types/testdata/fixedbugs/issue51360.go b/src/internal/types/testdata/fixedbugs/issue51360.go index 1b9c45a9..1798a4ab 100644 --- a/src/internal/types/testdata/fixedbugs/issue51360.go +++ b/src/internal/types/testdata/fixedbugs/issue51360.go @@ -5,9 +5,9 @@ package p func _() { - len.Println /* ERROR "cannot select on len" */ - len.Println /* ERROR "cannot select on len" */ () - _ = len.Println /* ERROR "cannot select on len" */ + len.Println /* ERROR "invalid use of len (built-in) in selector expression" */ + len.Println /* ERROR "invalid use of len (built-in) in selector expression" */ () + _ = len.Println /* ERROR "invalid use of len (built-in) in selector expression" */ _ = len /* ERROR "cannot index len" */ [0] _ = *len /* ERROR "cannot indirect len" */ } diff --git a/src/internal/types/testdata/fixedbugs/issue51503.go b/src/internal/types/testdata/fixedbugs/issue51503.go new file mode 100644 index 00000000..90a42562 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue51503.go @@ -0,0 +1,31 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type T[T any] struct{} + +// The inner T in T[T] must not conflict with the receiver base type T. +func (T[T]) m1() {} + +// The receiver parameter r is declared after the receiver type parameter +// r in T[r]. An error is expected for the receiver parameter. +func (r /* ERROR "r redeclared" */ T[r]) m2() {} + +type C any + +// The scope of type parameter C starts after the type name (_) +// because we want to be able to use type parameters in the type +// parameter list. Hence, the 2nd C in the type parameter list below +// refers to the first C. Since constraints cannot be type parameters +// this is an error. +type _[C C /* ERROR "cannot use a type parameter as constraint" */] struct{} + +// Same issue here. +func _[C C /* ERROR "cannot use a type parameter as constraint" */]() {} + +// The scope of ordinary parameter C starts after the function signature. +// Therefore, the 2nd C in the parameter list below refers to the type C. +// This code is correct. +func _(C C) {} // okay diff --git a/src/internal/types/testdata/fixedbugs/issue53535.go b/src/internal/types/testdata/fixedbugs/issue53535.go new file mode 100644 index 00000000..127b8a8b --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue53535.go @@ -0,0 +1,35 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +import "io" + +// test using struct with invalid embedded field +var _ io.Writer = W{} // no error expected here because W has invalid embedded field + +type W struct { + *bufio /* ERROR "undefined: bufio" */ .Writer +} + +// test using an invalid type +var _ interface{ m() } = &M{} // no error expected here because M is invalid + +type M undefined // ERROR "undefined: undefined" + +// test using struct with invalid embedded field and containing a self-reference (cycle) +var _ interface{ m() } = &S{} // no error expected here because S is invalid + +type S struct { + *S + undefined // ERROR "undefined: undefined" +} + +// test using a generic struct with invalid embedded field and containing a self-reference (cycle) +var _ interface{ m() } = &G[int]{} // no error expected here because S is invalid + +type G[P any] struct { + *G[P] + undefined // ERROR "undefined: undefined" +} diff --git a/src/internal/types/testdata/fixedbugs/issue60377.go b/src/internal/types/testdata/fixedbugs/issue60377.go index b754f89d..17a9deb6 100644 --- a/src/internal/types/testdata/fixedbugs/issue60377.go +++ b/src/internal/types/testdata/fixedbugs/issue60377.go @@ -57,7 +57,7 @@ func _() { var x S[int] g4(x) // we can infer int for P g4[int](x) // int is the correct type argument - g4[string](x /* ERROR "cannot use x (variable of type S[int]) as S[string] value in argument to g4[string]" */) + g4[string](x /* ERROR "cannot use x (variable of struct type S[int]) as S[string] value in argument to g4[string]" */) } // This is similar to the first example but here T1 is a component diff --git a/src/internal/types/testdata/fixedbugs/issue62157.go b/src/internal/types/testdata/fixedbugs/issue62157.go index c44f921f..67a110df 100644 --- a/src/internal/types/testdata/fixedbugs/issue62157.go +++ b/src/internal/types/testdata/fixedbugs/issue62157.go @@ -90,7 +90,7 @@ func _() { B = f(B, b, a) // verify type error - A = f /* ERROR "cannot use f(B, b, a) (value of type namedB) as namedA value in assignment" */ (B, b, a) + A = f /* ERROR "cannot use f(B, b, a) (value of chan type namedB) as namedA value in assignment" */ (B, b, a) } // Test case 4: some more combinations diff --git a/src/internal/types/testdata/fixedbugs/issue66751.go b/src/internal/types/testdata/fixedbugs/issue66751.go new file mode 100644 index 00000000..5a64b4dc --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue66751.go @@ -0,0 +1,62 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type S struct{} + +func (*S) m(int) {} + +func f[A interface { + ~*B + m(C) +}, B, C any]() { +} + +var _ = f[*S] // must be able to infer all remaining type arguments + +// original test case from issue + +type ptrTo[A any] interface{ ~*A } +type hasFoo[A any] interface{ foo(A) } +type both[A, B any] interface { + ptrTo[A] + hasFoo[B] +} + +type fooer[A any] struct{} + +func (f *fooer[A]) foo(A) {} + +func withPtr[A ptrTo[B], B any]() {} +func withFoo[A hasFoo[B], B any]() {} +func withBoth[A both[B, C], B, C any]() {} + +func _() { + withPtr[*fooer[int]]() // ok + withFoo[*fooer[int]]() // ok + withBoth[*fooer[int]]() // should be able to infer C +} + +// related test case reported in issue + +type X struct{} + +func (x X) M() int { return 42 } + +func CallM1[T interface{ M() R }, R any](t T) R { + return t.M() +} + +func CallM2[T interface { + X + M() R +}, R any](t T) R { + return t.M() +} + +func _() { + CallM1(X{}) // ok + CallM2(X{}) // should be able to infer R +} diff --git a/src/internal/types/testdata/fixedbugs/issue68184.go b/src/internal/types/testdata/fixedbugs/issue68184.go new file mode 100644 index 00000000..9c77365a --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue68184.go @@ -0,0 +1,38 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type VeryLongStruct struct { + A1 int + A2 int + A3 int + A4 int + A5 int + A6 int + A7 int + A8 int + A9 int + A10 int + A11 int + A12 int + A13 int + A14 int + A15 int + A16 int + A17 int + A18 int + A19 int + A20 int +} + +func _() { + // The error messages in both these cases should print the + // struct name rather than the struct's underlying type. + + var x VeryLongStruct + x.B2 /* ERROR "x.B2 undefined (type VeryLongStruct has no field or method B2)" */ = false + + _ = []VeryLongStruct{{B2 /* ERROR "unknown field B2 in struct literal of type VeryLongStruct" */ : false}} +} diff --git a/src/internal/types/testdata/fixedbugs/issue69576.go b/src/internal/types/testdata/fixedbugs/issue69576.go new file mode 100644 index 00000000..97e03dfa --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue69576.go @@ -0,0 +1,11 @@ +// -goexperiment=aliastypeparams -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A[P int] = struct{} + +var _ A[string /* ERROR "string does not satisfy int (string missing in int)" */] diff --git a/src/internal/types/testdata/fixedbugs/issue69955.go b/src/internal/types/testdata/fixedbugs/issue69955.go new file mode 100644 index 00000000..68ddf410 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue69955.go @@ -0,0 +1,42 @@ +// -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +import "math/big" + +type ( + S struct{} + N int + + A = S + B = int + C = N +) + +var ( + i int + s S + n N + a A + b B + c C + w big.Word +) + +const ( + _ = i // ERROR "i (variable of type int) is not constant" + _ = s // ERROR "s (variable of struct type S) is not constant" + _ = struct /* ERROR "struct{}{} (value of type struct{}) is not constant" */ {}{} + _ = n // ERROR "n (variable of int type N) is not constant" + + _ = a // ERROR "a (variable of struct type A) is not constant" + _ = b // ERROR "b (variable of int type B) is not constant" + _ = c // ERROR "c (variable of int type C) is not constant" + _ = w // ERROR "w (variable of uint type big.Word) is not constant" +) + +var _ int = w /* ERROR "cannot use w + 1 (value of uint type big.Word) as int value in variable declaration" */ + 1 diff --git a/src/internal/types/testdata/fixedbugs/issue70150.go b/src/internal/types/testdata/fixedbugs/issue70150.go new file mode 100644 index 00000000..5baf4a66 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue70150.go @@ -0,0 +1,15 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _() { + var values []int + vf(values /* ERROR "(variable of type []int) as string value" */) + vf(values...) /* ERROR "have ([]int...)\n\twant (string, ...int)" */ + vf("ab", "cd", values /* ERROR "have (string, string, []int...)\n\twant (string, ...int)" */ ...) +} + +func vf(method string, values ...int) { +} diff --git a/src/internal/types/testdata/fixedbugs/issue70417.go b/src/internal/types/testdata/fixedbugs/issue70417.go new file mode 100644 index 00000000..74bdea3b --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue70417.go @@ -0,0 +1,58 @@ +// -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type T[P any] struct{} + +// A0 +type A0 = T[int] +type B0 = *T[int] + +func (A0 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (*A0 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (B0 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} + +// A1 +type A1[P any] = T[P] +type B1[P any] = *T[P] + +func (A1 /* ERROR "cannot define new methods on generic alias type A1[P any]" */ [P]) m() {} +func (*A1 /* ERROR "cannot define new methods on generic alias type A1[P any]" */ [P]) m() {} +func (B1 /* ERROR "cannot define new methods on generic alias type B1[P any]" */ [P]) m() {} + +// A2 +type A2[P any] = T[int] +type B2[P any] = *T[int] + +func (A2 /* ERROR "cannot define new methods on generic alias type A2[P any]" */ [P]) m() {} +func (*A2 /* ERROR "cannot define new methods on generic alias type A2[P any]" */ [P]) m() {} +func (B2 /* ERROR "cannot define new methods on generic alias type B2[P any]" */ [P]) m() {} + +// A3 +type A3 = T[int] +type B3 = *T[int] + +func (A3 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (*A3 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (B3 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} + +// A4 +type A4 = T // ERROR "cannot use generic type T[P any] without instantiation" +type B4 = *T // ERROR "cannot use generic type T[P any] without instantiation" + +func (A4[P]) m1() {} // don't report a follow-on error on A4 +func (*A4[P]) m2() {} // don't report a follow-on error on A4 +func (B4[P]) m3() {} // don't report a follow-on error on B4 + +// instantiation in the middle of an alias chain +type S struct{} +type C0 = S +type C1[P any] = C0 +type C2 = *C1[int] + +func (C2 /* ERROR "cannot define new methods on instantiated type C1[int]" */) m() {} +func (*C2 /* ERROR "cannot define new methods on instantiated type C1[int]" */) m() {} diff --git a/src/internal/types/testdata/fixedbugs/issue70526.go b/src/internal/types/testdata/fixedbugs/issue70526.go new file mode 100644 index 00000000..56b20bfc --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue70526.go @@ -0,0 +1,13 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f(...any) + +func _(x int, s []int) { + f(0, x /* ERROR "have (number, int...)\n\twant (...any)" */ ...) + f(0, s /* ERROR "have (number, []int...)\n\twant (...any)" */ ...) + f(0, 0 /* ERROR "have (number, number...)\n\twant (...any)" */ ...) +} diff --git a/src/internal/types/testdata/fixedbugs/issue71131.go b/src/internal/types/testdata/fixedbugs/issue71131.go new file mode 100644 index 00000000..8e7575b0 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue71131.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _() { + type Bool bool + for range func /* ERROR "yield func returns user-defined boolean, not bool" */ (func() Bool) {} { + } + for range func /* ERROR "yield func returns user-defined boolean, not bool" */ (func(int) Bool) {} { + } + for range func /* ERROR "yield func returns user-defined boolean, not bool" */ (func(int, string) Bool) {} { + } +} diff --git a/src/internal/types/testdata/fixedbugs/issue71198.go b/src/internal/types/testdata/fixedbugs/issue71198.go new file mode 100644 index 00000000..479f8e2b --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue71198.go @@ -0,0 +1,16 @@ +// -gotypesalias=1 + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A[_ any] = any + +// This must not panic; also the error message must match the style for non-alias types, below. +func _[_ A /* ERROR "too many type arguments for type A: have 2, want 1" */ [int, string]]() {} + +type T[_ any] any + +func _[_ T /* ERROR "too many type arguments for type T: have 2, want 1" */ [int, string]]() {} diff --git a/src/internal/types/testdata/fixedbugs/issue71284.go b/src/internal/types/testdata/fixedbugs/issue71284.go new file mode 100644 index 00000000..4b73087a --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue71284.go @@ -0,0 +1,10 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package A + +type ( + _ = A + A /* ERROR "invalid recursive type: A refers to itself" */ = A +) diff --git a/src/internal/types/testdata/spec/comparisons.go b/src/internal/types/testdata/spec/comparisons.go index 492890e4..dd92d99b 100644 --- a/src/internal/types/testdata/spec/comparisons.go +++ b/src/internal/types/testdata/spec/comparisons.go @@ -108,13 +108,13 @@ func _[ _ = c == nil _ = b < b - _ = a /* ERROR "type parameter A is not comparable with <" */ < a - _ = l /* ERROR "type parameter L is not comparable with <" */ < l - _ = s /* ERROR "type parameter S is not comparable with <" */ < s - _ = p /* ERROR "type parameter P is not comparable with <" */ < p - _ = f /* ERROR "type parameter F is not comparable with <" */ < f - _ = i /* ERROR "type parameter I is not comparable with <" */ < i - _ = j /* ERROR "type parameter J is not comparable with <" */ < j - _ = m /* ERROR "type parameter M is not comparable with <" */ < m - _ = c /* ERROR "type parameter C is not comparable with <" */ < c + _ = a /* ERROR "type parameter A cannot use operator <" */ < a + _ = l /* ERROR "type parameter L cannot use operator <" */ < l + _ = s /* ERROR "type parameter S cannot use operator <" */ < s + _ = p /* ERROR "type parameter P cannot use operator <" */ < p + _ = f /* ERROR "type parameter F cannot use operator <" */ < f + _ = i /* ERROR "type parameter I cannot use operator <" */ < i + _ = j /* ERROR "type parameter J cannot use operator <" */ < j + _ = m /* ERROR "type parameter M cannot use operator <" */ < m + _ = c /* ERROR "type parameter C cannot use operator <" */ < c } diff --git a/src/internal/types/testdata/spec/range.go b/src/internal/types/testdata/spec/range.go index 9e32256f..c0f57947 100644 --- a/src/internal/types/testdata/spec/range.go +++ b/src/internal/types/testdata/spec/range.go @@ -5,7 +5,7 @@ package p type MyInt int32 -type MyBool bool +type MyBool = bool // TODO(gri) remove alias declaration - see go.dev/issues/71131, go.dev/issues/71164 type MyString string type MyFunc1 func(func(int) bool) type MyFunc2 func(int) bool @@ -102,11 +102,11 @@ func test() { for mi, ms := range f8 { _, _ = mi, ms } - for i /* ERROR "cannot use i (value of type MyInt) as int value in assignment" */, s /* ERROR "cannot use s (value of type MyString) as string value in assignment" */ = range f8 { + for i /* ERROR "cannot use i (value of int32 type MyInt) as int value in assignment" */, s /* ERROR "cannot use s (value of string type MyString) as string value in assignment" */ = range f8 { _, _ = mi, ms } for mi, ms := range f8 { - i, s = mi /* ERROR "cannot use mi (variable of type MyInt) as int value in assignment" */, ms /* ERROR "cannot use ms (variable of type MyString) as string value in assignment" */ + i, s = mi /* ERROR "cannot use mi (variable of int32 type MyInt) as int value in assignment" */, ms /* ERROR "cannot use ms (variable of string type MyString) as string value in assignment" */ } for mi, ms = range f8 { _, _ = mi, ms diff --git a/src/internal/types/testdata/spec/range_int.go b/src/internal/types/testdata/spec/range_int.go index 766736cc..db3a78ff 100644 --- a/src/internal/types/testdata/spec/range_int.go +++ b/src/internal/types/testdata/spec/range_int.go @@ -44,7 +44,7 @@ func _() { for i, j /* ERROR "range over 10 (untyped int constant) permits only one iteration variable" */ := range 10 { _, _ = i, j } - for i = range MyInt /* ERROR "cannot use MyInt(10) (constant 10 of type MyInt) as int value in range clause" */ (10) { + for i = range MyInt /* ERROR "cannot use MyInt(10) (constant 10 of int32 type MyInt) as int value in range clause" */ (10) { _ = i } for mi := range MyInt(10) { diff --git a/src/internal/types/testdata/spec/receivers.go b/src/internal/types/testdata/spec/receivers.go new file mode 100644 index 00000000..010c5511 --- /dev/null +++ b/src/internal/types/testdata/spec/receivers.go @@ -0,0 +1,14 @@ +// -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package receivers + +// TODO(gri) add more tests checking the various restrictions on receivers + +type G[P any] struct{} +type A[P any] = G[P] + +func (a A /* ERROR "cannot define new methods on generic alias type A[P any]" */ [P]) m() {} diff --git a/src/internal/weak/pointer.go b/src/internal/weak/pointer.go deleted file mode 100644 index 8e05af2d..00000000 --- a/src/internal/weak/pointer.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -The weak package is a package for managing weak pointers. - -Weak pointers are pointers that explicitly do not keep a value live and -must be queried for a regular Go pointer. -The result of such a query may be observed as nil at any point after a -weakly-pointed-to object becomes eligible for reclamation by the garbage -collector. -More specifically, weak pointers become nil as soon as the garbage collector -identifies that the object is unreachable, before it is made reachable -again by a finalizer. -In terms of the C# language, these semantics are roughly equivalent to the -the semantics of "short" weak references. -In terms of the Java language, these semantics are roughly equivalent to the -semantics of the WeakReference type. - -Using go:linkname to access this package and the functions it references -is explicitly forbidden by the toolchain because the semantics of this -package have not gone through the proposal process. By exposing this -functionality, we risk locking in the existing semantics due to Hyrum's Law. - -If you believe you have a good use-case for weak references not already -covered by the standard library, file a proposal issue at -https://github.com/golang/go/issues instead of relying on this package. -*/ -package weak - -import ( - "internal/abi" - "runtime" - "unsafe" -) - -// Pointer is a weak pointer to a value of type T. -// -// This value is comparable is guaranteed to compare equal if the pointers -// that they were created from compare equal. This property is retained even -// after the object referenced by the pointer used to create a weak reference -// is reclaimed. -// -// If multiple weak pointers are made to different offsets within same object -// (for example, pointers to different fields of the same struct), those pointers -// will not compare equal. -// If a weak pointer is created from an object that becomes reachable again due -// to a finalizer, that weak pointer will not compare equal with weak pointers -// created before it became unreachable. -type Pointer[T any] struct { - u unsafe.Pointer -} - -// Make creates a weak pointer from a strong pointer to some value of type T. -func Make[T any](ptr *T) Pointer[T] { - // Explicitly force ptr to escape to the heap. - ptr = abi.Escape(ptr) - - var u unsafe.Pointer - if ptr != nil { - u = runtime_registerWeakPointer(unsafe.Pointer(ptr)) - } - runtime.KeepAlive(ptr) - return Pointer[T]{u} -} - -// Strong creates a strong pointer from the weak pointer. -// Returns nil if the original value for the weak pointer was reclaimed by -// the garbage collector. -// If a weak pointer points to an object with a finalizer, then Strong will -// return nil as soon as the object's finalizer is queued for execution. -func (p Pointer[T]) Strong() *T { - return (*T)(runtime_makeStrongFromWeak(p.u)) -} - -// Implemented in runtime. - -//go:linkname runtime_registerWeakPointer -func runtime_registerWeakPointer(unsafe.Pointer) unsafe.Pointer - -//go:linkname runtime_makeStrongFromWeak -func runtime_makeStrongFromWeak(unsafe.Pointer) unsafe.Pointer diff --git a/src/internal/xcoff/file_test.go b/src/internal/xcoff/file_test.go index a6722e94..d1f10d6b 100644 --- a/src/internal/xcoff/file_test.go +++ b/src/internal/xcoff/file_test.go @@ -6,6 +6,7 @@ package xcoff import ( "reflect" + "slices" "testing" ) @@ -87,7 +88,7 @@ func TestOpen(t *testing.T) { if err != nil { t.Error(err) } - if !reflect.DeepEqual(tl, fl) { + if !slices.Equal(tl, fl) { t.Errorf("open %s: loader import = %v, want %v", tt.file, tl, fl) } } diff --git a/src/internal/zstd/zstd.go b/src/internal/zstd/zstd.go index 0370f601..d4eac399 100644 --- a/src/internal/zstd/zstd.go +++ b/src/internal/zstd/zstd.go @@ -358,30 +358,12 @@ func (r *Reader) skipFrame() error { return nil } - var skip []byte - const chunk = 1 << 20 // 1M - for size >= chunk { - if len(skip) == 0 { - skip = make([]byte, chunk) - } - if _, err := io.ReadFull(r.r, skip); err != nil { - return r.wrapNonEOFError(relativeOffset, err) - } - relativeOffset += chunk - size -= chunk + n, err := io.CopyN(io.Discard, r.r, int64(size)) + relativeOffset += int(n) + if err != nil { + return r.wrapNonEOFError(relativeOffset, err) } - if size > 0 { - if len(skip) == 0 { - skip = make([]byte, size) - } - if _, err := io.ReadFull(r.r, skip); err != nil { - return r.wrapNonEOFError(relativeOffset, err) - } - relativeOffset += int(size) - } - r.blockOffset += int64(relativeOffset) - return nil } diff --git a/src/io/fs/fs.go b/src/io/fs/fs.go index 6891d75a..9e19e6a7 100644 --- a/src/io/fs/fs.go +++ b/src/io/fs/fs.go @@ -26,6 +26,7 @@ import ( // correctness. type FS interface { // Open opens the named file. + // [File.Close] must be called to release any associated resources. // // When Open returns an error, it should be of type *PathError // with the Op field set to "open", the Path field set to name, @@ -43,7 +44,7 @@ type FS interface { // Path names passed to open are UTF-8-encoded, // unrooted, slash-separated sequences of path elements, like “x/y/z”. // Path names must not contain an element that is “.” or “..” or the empty string, -// except for the special case that the root directory is named “.”. +// except for the special case that the name "." may be used for the root directory. // Paths must not start or end with a slash: “/x” and “x/” are invalid. // // Note that paths are slash-separated on all systems, even Windows. diff --git a/src/io/fs/walk_test.go b/src/io/fs/walk_test.go index 40f4e1ab..a5fc715e 100644 --- a/src/io/fs/walk_test.go +++ b/src/io/fs/walk_test.go @@ -9,7 +9,7 @@ import ( "os" pathpkg "path" "path/filepath" - "reflect" + "slices" "testing" "testing/fstest" ) @@ -86,16 +86,7 @@ func mark(entry DirEntry, err error, errors *[]error, clear bool) error { } func TestWalkDir(t *testing.T) { - tmpDir := t.TempDir() - - origDir, err := os.Getwd() - if err != nil { - t.Fatal("finding working dir:", err) - } - if err = os.Chdir(tmpDir); err != nil { - t.Fatal("entering temp dir:", err) - } - defer os.Chdir(origDir) + t.Chdir(t.TempDir()) fsys := makeTree() errors := make([]error, 0, 10) @@ -104,7 +95,7 @@ func TestWalkDir(t *testing.T) { return mark(entry, err, &errors, clear) } // Expect no errors. - err = WalkDir(fsys, ".", markFn) + err := WalkDir(fsys, ".", markFn) if err != nil { t.Fatalf("no error expected, found: %s", err) } @@ -145,7 +136,7 @@ func TestIssue51617(t *testing.T) { t.Fatal(err) } want := []string{".", "a", "a/bad", "a/next"} - if !reflect.DeepEqual(saw, want) { + if !slices.Equal(saw, want) { t.Errorf("got directories %v, want %v", saw, want) } } diff --git a/src/io/io_test.go b/src/io/io_test.go index 9491ffae..38bec824 100644 --- a/src/io/io_test.go +++ b/src/io/io_test.go @@ -505,7 +505,7 @@ func TestNopCloserWriterToForwarding(t *testing.T) { func TestOffsetWriter_Seek(t *testing.T) { tmpfilename := "TestOffsetWriter_Seek" tmpfile, err := os.CreateTemp(t.TempDir(), tmpfilename) - if err != nil || tmpfile == nil { + if err != nil { t.Fatalf("CreateTemp(%s) failed: %v", tmpfilename, err) } defer tmpfile.Close() @@ -564,15 +564,12 @@ func TestOffsetWriter_Seek(t *testing.T) { func TestOffsetWriter_WriteAt(t *testing.T) { const content = "0123456789ABCDEF" contentSize := int64(len(content)) - tmpdir, err := os.MkdirTemp(t.TempDir(), "TestOffsetWriter_WriteAt") - if err != nil { - t.Fatal(err) - } + tmpdir := t.TempDir() work := func(off, at int64) { position := fmt.Sprintf("off_%d_at_%d", off, at) tmpfile, err := os.CreateTemp(tmpdir, position) - if err != nil || tmpfile == nil { + if err != nil { t.Fatalf("CreateTemp(%s) failed: %v", position, err) } defer tmpfile.Close() @@ -642,7 +639,7 @@ func TestOffsetWriter_Write(t *testing.T) { makeOffsetWriter := func(name string) (*OffsetWriter, *os.File) { tmpfilename := "TestOffsetWriter_Write_" + name tmpfile, err := os.CreateTemp(tmpdir, tmpfilename) - if err != nil || tmpfile == nil { + if err != nil { t.Fatalf("CreateTemp(%s) failed: %v", tmpfilename, err) } return NewOffsetWriter(tmpfile, 0), tmpfile diff --git a/src/iter/iter.go b/src/iter/iter.go index 14fd8f81..e765378e 100644 --- a/src/iter/iter.go +++ b/src/iter/iter.go @@ -28,7 +28,22 @@ or index-value pairs. Yield returns true if the iterator should continue with the next element in the sequence, false if it should stop. -Iterator functions are most often called by a range loop, as in: +For instance, [maps.Keys] returns an iterator that produces the sequence +of keys of the map m, implemented as follows: + + func Keys[Map ~map[K]V, K comparable, V any](m Map) iter.Seq[K] { + return func(yield func(K) bool) { + for k := range m { + if !yield(k) { + return + } + } + } + } + +Further examples can be found in [The Go Blog: Range Over Function Types]. + +Iterator functions are most often called by a [range loop], as in: func PrintAll[V any](seq iter.Seq[V]) { for v := range seq { @@ -187,6 +202,9 @@ And then a client could delete boring values from the tree using: p.Delete() } } + +[The Go Blog: Range Over Function Types]: https://go.dev/blog/range-functions +[range loop]: https://go.dev/ref/spec#For_range */ package iter diff --git a/src/iter/pull_test.go b/src/iter/pull_test.go index 449edee0..e9e3bdad 100644 --- a/src/iter/pull_test.go +++ b/src/iter/pull_test.go @@ -193,7 +193,11 @@ func doDoubleNext2() Seq2[int, int] { } func TestPullDoubleYield(t *testing.T) { - _, stop := Pull(storeYield()) + next, stop := Pull(storeYield()) + next() + if yieldSlot == nil { + t.Fatal("yield failed") + } defer func() { if recover() != nil { yieldSlot = nil @@ -218,7 +222,11 @@ func storeYield() Seq[int] { var yieldSlot func(int) bool func TestPullDoubleYield2(t *testing.T) { - _, stop := Pull2(storeYield2()) + next, stop := Pull2(storeYield2()) + next() + if yieldSlot2 == nil { + t.Fatal("yield failed") + } defer func() { if recover() != nil { yieldSlot2 = nil diff --git a/src/log/slog/attr_test.go b/src/log/slog/attr_test.go index e01447cf..1f2ce262 100644 --- a/src/log/slog/attr_test.go +++ b/src/log/slog/attr_test.go @@ -5,12 +5,16 @@ package slog import ( + "internal/asan" "internal/testenv" "testing" "time" ) func TestAttrNoAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates with -asan") + } testenv.SkipIfOptimizationOff(t) // Assign values just to make sure the compiler doesn't optimize away the statements. var ( diff --git a/src/log/slog/example_discard_test.go b/src/log/slog/example_discard_test.go new file mode 100644 index 00000000..3e3e37b1 --- /dev/null +++ b/src/log/slog/example_discard_test.go @@ -0,0 +1,27 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog_test + +import ( + "log/slog" + "log/slog/internal/slogtest" + "os" +) + +func Example_discardHandler() { + // A slog.TextHandler can output log messages. + logger1 := slog.New(slog.NewTextHandler( + os.Stdout, + &slog.HandlerOptions{ReplaceAttr: slogtest.RemoveTime}, + )) + logger1.Info("message 1") + + // A slog.DiscardHandler will discard all messages. + logger2 := slog.New(slog.DiscardHandler) + logger2.Info("message 2") + + // Output: + // level=INFO msg="message 1" +} diff --git a/src/log/slog/handler.go b/src/log/slog/handler.go index 2ff85b58..1ca4f9db 100644 --- a/src/log/slog/handler.go +++ b/src/log/slog/handler.go @@ -602,3 +602,14 @@ func appendRFC3339Millis(b []byte, t time.Time) []byte { b = append(b[:n+prefixLen], b[n+prefixLen+1:]...) // drop the 4th digit return b } + +// DiscardHandler discards all log output. +// DiscardHandler.Enabled returns false for all Levels. +var DiscardHandler Handler = discardHandler{} + +type discardHandler struct{} + +func (dh discardHandler) Enabled(context.Context, Level) bool { return false } +func (dh discardHandler) Handle(context.Context, Record) error { return nil } +func (dh discardHandler) WithAttrs(attrs []Attr) Handler { return dh } +func (dh discardHandler) WithGroup(name string) Handler { return dh } diff --git a/src/log/slog/handler_test.go b/src/log/slog/handler_test.go index 8ce34526..d34025f1 100644 --- a/src/log/slog/handler_test.go +++ b/src/log/slog/handler_test.go @@ -11,6 +11,7 @@ import ( "context" "encoding/json" "io" + "os" "path/filepath" "slices" "strconv" @@ -711,3 +712,23 @@ func BenchmarkWriteTime(b *testing.B) { buf = appendRFC3339Millis(buf[:0], tm) } } + +func TestDiscardHandler(t *testing.T) { + ctx := context.Background() + stdout, stderr := os.Stdout, os.Stderr + os.Stdout, os.Stderr = nil, nil // panic on write + t.Cleanup(func() { + os.Stdout, os.Stderr = stdout, stderr + }) + + // Just ensure nothing panics during normal usage + l := New(DiscardHandler) + l.Info("msg", "a", 1, "b", 2) + l.Debug("bg", Int("a", 1), "b", 2) + l.Warn("w", Duration("dur", 3*time.Second)) + l.Error("bad", "a", 1) + l.Log(ctx, LevelWarn+1, "w", Int("a", 1), String("b", "two")) + l.LogAttrs(ctx, LevelInfo+1, "a b c", Int("a", 1), String("b", "two")) + l.Info("info", "a", []Attr{Int("i", 1)}) + l.Info("info", "a", GroupValue(Int("i", 1))) +} diff --git a/src/log/slog/level.go b/src/log/slog/level.go index 7cddf4cf..2957585e 100644 --- a/src/log/slog/level.go +++ b/src/log/slog/level.go @@ -98,10 +98,16 @@ func (l *Level) UnmarshalJSON(data []byte) error { return l.parse(s) } -// MarshalText implements [encoding.TextMarshaler] +// AppendText implements [encoding.TextAppender] // by calling [Level.String]. +func (l Level) AppendText(b []byte) ([]byte, error) { + return append(b, l.String()...), nil +} + +// MarshalText implements [encoding.TextMarshaler] +// by calling [Level.AppendText]. func (l Level) MarshalText() ([]byte, error) { - return []byte(l.String()), nil + return l.AppendText(nil) } // UnmarshalText implements [encoding.TextUnmarshaler]. @@ -172,10 +178,16 @@ func (v *LevelVar) String() string { return fmt.Sprintf("LevelVar(%s)", v.Level()) } +// AppendText implements [encoding.TextAppender] +// by calling [Level.AppendText]. +func (v *LevelVar) AppendText(b []byte) ([]byte, error) { + return v.Level().AppendText(b) +} + // MarshalText implements [encoding.TextMarshaler] -// by calling [Level.MarshalText]. +// by calling [LevelVar.AppendText]. func (v *LevelVar) MarshalText() ([]byte, error) { - return v.Level().MarshalText() + return v.AppendText(nil) } // UnmarshalText implements [encoding.TextUnmarshaler] diff --git a/src/log/slog/level_test.go b/src/log/slog/level_test.go index 217a0d72..73be1126 100644 --- a/src/log/slog/level_test.go +++ b/src/log/slog/level_test.go @@ -89,6 +89,19 @@ func TestLevelMarshalText(t *testing.T) { } } +func TestLevelAppendText(t *testing.T) { + buf := make([]byte, 4, 16) + want := LevelWarn - 3 + wantData := []byte("\x00\x00\x00\x00INFO+1") + data, err := want.AppendText(buf) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, wantData) { + t.Errorf("got %s, want %s", string(data), string(wantData)) + } +} + func TestLevelParse(t *testing.T) { for _, test := range []struct { in string @@ -162,6 +175,23 @@ func TestLevelVarMarshalText(t *testing.T) { } } +func TestLevelVarAppendText(t *testing.T) { + var v LevelVar + v.Set(LevelWarn) + buf := make([]byte, 4, 16) + data, err := v.AppendText(buf) + if err != nil { + t.Fatal(err) + } + var v2 LevelVar + if err := v2.UnmarshalText(data[4:]); err != nil { + t.Fatal(err) + } + if g, w := v2.Level(), LevelWarn; g != w { + t.Errorf("got %s, want %s", g, w) + } +} + func TestLevelVarFlag(t *testing.T) { fs := flag.NewFlagSet("test", flag.ContinueOnError) v := &LevelVar{} diff --git a/src/log/slog/logger.go b/src/log/slog/logger.go index 10aa6a2b..961e0cd2 100644 --- a/src/log/slog/logger.go +++ b/src/log/slog/logger.go @@ -5,6 +5,7 @@ package slog import ( + "bytes" "context" "log" loginternal "log/internal" @@ -96,9 +97,7 @@ func (w *handlerWriter) Write(buf []byte) (int, error) { // Remove final newline. origLen := len(buf) // Report that the entire buf was written. - if len(buf) > 0 && buf[len(buf)-1] == '\n' { - buf = buf[:len(buf)-1] - } + buf = bytes.TrimSuffix(buf, []byte{'\n'}) r := NewRecord(time.Now(), level, string(buf), pc) return origLen, w.h.Handle(context.Background(), r) } diff --git a/src/log/slog/logger_test.go b/src/log/slog/logger_test.go index bb1c8a16..0f1b2113 100644 --- a/src/log/slog/logger_test.go +++ b/src/log/slog/logger_test.go @@ -7,6 +7,8 @@ package slog import ( "bytes" "context" + "internal/asan" + "internal/msan" "internal/race" "internal/testenv" "io" @@ -229,7 +231,7 @@ func TestCallDepth(t *testing.T) { func TestAlloc(t *testing.T) { ctx := context.Background() - dl := New(discardHandler{}) + dl := New(discardTestHandler{}) defer SetDefault(Default()) // restore SetDefault(dl) @@ -256,7 +258,7 @@ func TestAlloc(t *testing.T) { }) }) t.Run("2 pairs disabled inline", func(t *testing.T) { - l := New(discardHandler{disabled: true}) + l := New(DiscardHandler) s := "abc" i := 2000 wantAllocs(t, 2, func() { @@ -267,7 +269,7 @@ func TestAlloc(t *testing.T) { }) }) t.Run("2 pairs disabled", func(t *testing.T) { - l := New(discardHandler{disabled: true}) + l := New(DiscardHandler) s := "abc" i := 2000 wantAllocs(t, 0, func() { @@ -303,7 +305,7 @@ func TestAlloc(t *testing.T) { }) }) t.Run("attrs3 disabled", func(t *testing.T) { - logger := New(discardHandler{disabled: true}) + logger := New(DiscardHandler) wantAllocs(t, 0, func() { logger.LogAttrs(ctx, LevelInfo, "hello", Int("a", 1), String("b", "two"), Duration("c", time.Second)) }) @@ -566,18 +568,17 @@ func (c *captureHandler) clear() { c.r = Record{} } -type discardHandler struct { - disabled bool - attrs []Attr +type discardTestHandler struct { + attrs []Attr } -func (d discardHandler) Enabled(context.Context, Level) bool { return !d.disabled } -func (discardHandler) Handle(context.Context, Record) error { return nil } -func (d discardHandler) WithAttrs(as []Attr) Handler { +func (d discardTestHandler) Enabled(context.Context, Level) bool { return true } +func (discardTestHandler) Handle(context.Context, Record) error { return nil } +func (d discardTestHandler) WithAttrs(as []Attr) Handler { d.attrs = concat(d.attrs, as) return d } -func (h discardHandler) WithGroup(name string) Handler { +func (h discardTestHandler) WithGroup(name string) Handler { return h } @@ -644,8 +645,8 @@ func callerPC(depth int) uintptr { } func wantAllocs(t *testing.T, want int, f func()) { - if race.Enabled { - t.Skip("skipping test in race mode") + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("skipping test in race, asan, and msan modes") } testenv.SkipIfOptimizationOff(t) t.Helper() diff --git a/src/log/slog/value_test.go b/src/log/slog/value_test.go index 3e191589..4f405938 100644 --- a/src/log/slog/value_test.go +++ b/src/log/slog/value_test.go @@ -6,6 +6,7 @@ package slog import ( "fmt" + "internal/asan" "reflect" "strings" "testing" @@ -86,6 +87,10 @@ func TestValueString(t *testing.T) { } func TestValueNoAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } + // Assign values just to make sure the compiler doesn't optimize away the statements. var ( i int64 diff --git a/src/make.bash b/src/make.bash index 814b7e87..b67ae152 100755 --- a/src/make.bash +++ b/src/make.bash @@ -64,14 +64,14 @@ # timing information to this file. Useful for profiling where the # time goes when these scripts run. # -# GOROOT_BOOTSTRAP: A working Go tree >= Go 1.20.6 for bootstrap. +# GOROOT_BOOTSTRAP: A working Go tree >= Go 1.22.6 for bootstrap. # If $GOROOT_BOOTSTRAP/bin/go is missing, $(go env GOROOT) is -# tried for all "go" in $PATH. By default, one of $HOME/go1.20.6, -# $HOME/sdk/go1.20.6, or $HOME/go1.4, whichever exists, in that order. +# tried for all "go" in $PATH. By default, one of $HOME/go1.22.6, +# $HOME/sdk/go1.22.6, or $HOME/go1.4, whichever exists, in that order. # We still check $HOME/go1.4 to allow for build scripts that still hard-code # that name even though they put newer Go toolchains there. -bootgo=1.20.6 +bootgo=1.22.6 set -e @@ -128,13 +128,6 @@ do fi done -# Test for debian/kFreeBSD. -# cmd/dist will detect kFreeBSD as freebsd/$GOARCH, but we need to -# disable cgo manually. -if [[ "$(uname -s)" == "GNU/kFreeBSD" ]]; then - export CGO_ENABLED=0 -fi - # Clean old generated file that will cause problems in the build. rm -f ./runtime/runtime_defs.go diff --git a/src/make.bat b/src/make.bat index 53122cba..3b5a4663 100644 --- a/src/make.bat +++ b/src/make.bat @@ -85,7 +85,7 @@ for /f "tokens=*" %%g in ('where go 2^>nul') do ( ) ) -set bootgo=1.20.6 +set bootgo=1.22.6 if "x%GOROOT_BOOTSTRAP%"=="x" if exist "%HOMEDRIVE%%HOMEPATH%\go%bootgo%" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\go%bootgo% if "x%GOROOT_BOOTSTRAP%"=="x" if exist "%HOMEDRIVE%%HOMEPATH%\sdk\go%bootgo%" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\sdk\go%bootgo% if "x%GOROOT_BOOTSTRAP%"=="x" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\Go1.4 diff --git a/src/make.rc b/src/make.rc index 54250011..b3beb756 100755 --- a/src/make.rc +++ b/src/make.rc @@ -48,10 +48,10 @@ fn bootstrapenv { GOROOT=$GOROOT_BOOTSTRAP GO111MODULE=off GOENV=off GOOS=() GOARCH=() GOEXPERIMENT=() GOFLAGS=() $* } -bootgo = 1.20.6 +bootgo = 1.22.6 GOROOT = `{cd .. && pwd} goroot_bootstrap_set = 'true' -if(! ~ $#GOROOT_BOOTSTRAP 1){ +if(~ $"GOROOT_BOOTSTRAP ''){ goroot_bootstrap_set = 'false' GOROOT_BOOTSTRAP = $home/go1.4 for(d in sdk/go$bootgo go$bootgo) diff --git a/src/maps/example_test.go b/src/maps/example_test.go index 3d6b7d1b..0795941d 100644 --- a/src/maps/example_test.go +++ b/src/maps/example_test.go @@ -7,6 +7,7 @@ package maps_test import ( "fmt" "maps" + "slices" "strings" ) @@ -133,3 +134,60 @@ func ExampleEqualFunc() { // Output: // true } + +func ExampleAll() { + m1 := map[string]int{ + "one": 1, + "two": 2, + } + m2 := map[string]int{ + "one": 10, + } + maps.Insert(m2, maps.All(m1)) + fmt.Println("m2 is:", m2) + // Output: + // m2 is: map[one:1 two:2] +} + +func ExampleKeys() { + m1 := map[int]string{ + 1: "one", + 10: "Ten", + 1000: "THOUSAND", + } + keys := slices.Sorted(maps.Keys(m1)) + fmt.Println(keys) + // Output: + // [1 10 1000] +} + +func ExampleValues() { + m1 := map[int]string{ + 1: "one", + 10: "Ten", + 1000: "THOUSAND", + } + values := slices.Sorted(maps.Values(m1)) + fmt.Println(values) + // Output: + // [THOUSAND Ten one] +} + +func ExampleInsert() { + m1 := map[int]string{ + 1000: "THOUSAND", + } + s1 := []string{"zero", "one", "two", "three"} + maps.Insert(m1, slices.All(s1)) + fmt.Println("m1 is:", m1) + // Output: + // m1 is: map[0:zero 1:one 2:two 3:three 1000:THOUSAND] +} + +func ExampleCollect() { + s1 := []string{"zero", "one", "two", "three"} + m1 := maps.Collect(slices.All(s1)) + fmt.Println("m1 is:", m1) + // Output: + // m1 is: map[0:zero 1:one 2:two 3:three] +} diff --git a/src/maps/maps_test.go b/src/maps/maps_test.go index fa30fe8c..c8ee3cd0 100644 --- a/src/maps/maps_test.go +++ b/src/maps/maps_test.go @@ -48,7 +48,7 @@ func equalNaN[T comparable](v1, v2 T) bool { return v1 == v2 || (isNaN(v1) && isNaN(v2)) } -// equalStr compares ints and strings. +// equalIntStr compares ints and strings. func equalIntStr(v1 int, v2 string) bool { return strconv.Itoa(v1) == v2 } diff --git a/src/math/all_test.go b/src/math/all_test.go index af3c38c2..c253b7bc 100644 --- a/src/math/all_test.go +++ b/src/math/all_test.go @@ -899,8 +899,17 @@ var vfceilSC = []float64{ 0, Inf(1), NaN(), + 1<<52 - 1, + 1<<52 - 0.5, // largest fractional float64 + 1 << 52, + -1 << 52, + -1<<52 + 0.5, // smallest fractional float64 + -1<<52 + 1, + 1 << 53, + -1 << 53, } -var ceilSC = []float64{ + +var ceilBaseSC = []float64{ Inf(-1), Copysign(0, -1), 0, @@ -908,6 +917,39 @@ var ceilSC = []float64{ NaN(), } +var ceilSC = append(ceilBaseSC, + 1<<52-1, + 1<<52, + 1<<52, + -1<<52, + -1<<52+1, + -1<<52+1, + 1<<53, + -1<<53, +) + +var floorSC = append(ceilBaseSC, + 1<<52-1, + 1<<52-1, + 1<<52, + -1<<52, + -1<<52, + -1<<52+1, + 1<<53, + -1<<53, +) + +var truncSC = append(ceilBaseSC, + 1<<52-1, + 1<<52-1, + 1<<52, + -1<<52, + -1<<52+1, + -1<<52+1, + 1<<53, + -1<<53, +) + var vfcopysignSC = []float64{ Inf(-1), Inf(1), @@ -2489,8 +2531,8 @@ func TestFloor(t *testing.T) { } } for i := 0; i < len(vfceilSC); i++ { - if f := Floor(vfceilSC[i]); !alike(ceilSC[i], f) { - t.Errorf("Floor(%g) = %g, want %g", vfceilSC[i], f, ceilSC[i]) + if f := Floor(vfceilSC[i]); !alike(floorSC[i], f) { + t.Errorf("Floor(%g) = %g, want %g", vfceilSC[i], f, floorSC[i]) } } } @@ -3034,8 +3076,8 @@ func TestTrunc(t *testing.T) { } } for i := 0; i < len(vfceilSC); i++ { - if f := Trunc(vfceilSC[i]); !alike(ceilSC[i], f) { - t.Errorf("Trunc(%g) = %g, want %g", vfceilSC[i], f, ceilSC[i]) + if f := Trunc(vfceilSC[i]); !alike(truncSC[i], f) { + t.Errorf("Trunc(%g) = %g, want %g", vfceilSC[i], f, truncSC[i]) } } } diff --git a/src/math/big/arith_riscv64.s b/src/math/big/arith_riscv64.s index bad32497..069a4080 100644 --- a/src/math/big/arith_riscv64.s +++ b/src/math/big/arith_riscv64.s @@ -10,16 +10,288 @@ // arithmetic operations on vectors implemented in arith.go. TEXT ·addVV(SB),NOSPLIT,$0 - JMP ·addVV_g(SB) + MOV x+24(FP), X5 + MOV y+48(FP), X6 + MOV z+0(FP), X7 + MOV z_len+8(FP), X30 + + MOV $4, X28 + MOV $0, X29 // c = 0 + + BEQZ X30, done + BLTU X30, X28, loop1 + +loop4: + MOV 0(X5), X8 // x[0] + MOV 0(X6), X9 // y[0] + MOV 8(X5), X11 // x[1] + MOV 8(X6), X12 // y[1] + MOV 16(X5), X14 // x[2] + MOV 16(X6), X15 // y[2] + MOV 24(X5), X17 // x[3] + MOV 24(X6), X18 // y[3] + + ADD X8, X9, X21 // z[0] = x[0] + y[0] + SLTU X8, X21, X22 + ADD X21, X29, X10 // z[0] = x[0] + y[0] + c + SLTU X21, X10, X23 + ADD X22, X23, X29 // next c + + ADD X11, X12, X24 // z[1] = x[1] + y[1] + SLTU X11, X24, X25 + ADD X24, X29, X13 // z[1] = x[1] + y[1] + c + SLTU X24, X13, X26 + ADD X25, X26, X29 // next c + + ADD X14, X15, X21 // z[2] = x[2] + y[2] + SLTU X14, X21, X22 + ADD X21, X29, X16 // z[2] = x[2] + y[2] + c + SLTU X21, X16, X23 + ADD X22, X23, X29 // next c + + ADD X17, X18, X21 // z[3] = x[3] + y[3] + SLTU X17, X21, X22 + ADD X21, X29, X19 // z[3] = x[3] + y[3] + c + SLTU X21, X19, X23 + ADD X22, X23, X29 // next c + + MOV X10, 0(X7) // z[0] + MOV X13, 8(X7) // z[1] + MOV X16, 16(X7) // z[2] + MOV X19, 24(X7) // z[3] + + ADD $32, X5 + ADD $32, X6 + ADD $32, X7 + SUB $4, X30 + + BGEU X30, X28, loop4 + BEQZ X30, done + +loop1: + MOV 0(X5), X10 // x + MOV 0(X6), X11 // y + + ADD X10, X11, X12 // z = x + y + SLTU X10, X12, X14 + ADD X12, X29, X13 // z = x + y + c + SLTU X12, X13, X15 + ADD X14, X15, X29 // next c + + MOV X13, 0(X7) // z + + ADD $8, X5 + ADD $8, X6 + ADD $8, X7 + SUB $1, X30 + + BNEZ X30, loop1 + +done: + MOV X29, c+72(FP) // return c + RET TEXT ·subVV(SB),NOSPLIT,$0 - JMP ·subVV_g(SB) + MOV x+24(FP), X5 + MOV y+48(FP), X6 + MOV z+0(FP), X7 + MOV z_len+8(FP), X30 + + MOV $4, X28 + MOV $0, X29 // b = 0 + + BEQZ X30, done + BLTU X30, X28, loop1 + +loop4: + MOV 0(X5), X8 // x[0] + MOV 0(X6), X9 // y[0] + MOV 8(X5), X11 // x[1] + MOV 8(X6), X12 // y[1] + MOV 16(X5), X14 // x[2] + MOV 16(X6), X15 // y[2] + MOV 24(X5), X17 // x[3] + MOV 24(X6), X18 // y[3] + + SUB X9, X8, X21 // z[0] = x[0] - y[0] + SLTU X21, X8, X22 + SUB X29, X21, X10 // z[0] = x[0] - y[0] - b + SLTU X10, X21, X23 + ADD X22, X23, X29 // next b + + SUB X12, X11, X24 // z[1] = x[1] - y[1] + SLTU X24, X11, X25 + SUB X29, X24, X13 // z[1] = x[1] - y[1] - b + SLTU X13, X24, X26 + ADD X25, X26, X29 // next b + + SUB X15, X14, X21 // z[2] = x[2] - y[2] + SLTU X21, X14, X22 + SUB X29, X21, X16 // z[2] = x[2] - y[2] - b + SLTU X16, X21, X23 + ADD X22, X23, X29 // next b + + SUB X18, X17, X21 // z[3] = x[3] - y[3] + SLTU X21, X17, X22 + SUB X29, X21, X19 // z[3] = x[3] - y[3] - b + SLTU X19, X21, X23 + ADD X22, X23, X29 // next b + + MOV X10, 0(X7) // z[0] + MOV X13, 8(X7) // z[1] + MOV X16, 16(X7) // z[2] + MOV X19, 24(X7) // z[3] + + ADD $32, X5 + ADD $32, X6 + ADD $32, X7 + SUB $4, X30 + + BGEU X30, X28, loop4 + BEQZ X30, done + +loop1: + MOV 0(X5), X10 // x + MOV 0(X6), X11 // y + + SUB X11, X10, X12 // z = x - y + SLTU X12, X10, X14 + SUB X29, X12, X13 // z = x - y - b + SLTU X13, X12, X15 + ADD X14, X15, X29 // next b + + MOV X13, 0(X7) // z + + ADD $8, X5 + ADD $8, X6 + ADD $8, X7 + SUB $1, X30 + + BNEZ X30, loop1 + +done: + MOV X29, c+72(FP) // return b + RET TEXT ·addVW(SB),NOSPLIT,$0 - JMP ·addVW_g(SB) + MOV x+24(FP), X5 + MOV y+48(FP), X6 + MOV z+0(FP), X7 + MOV z_len+8(FP), X30 + + MOV $4, X28 + MOV X6, X29 // c = y + + BEQZ X30, done + BLTU X30, X28, loop1 + +loop4: + MOV 0(X5), X8 // x[0] + MOV 8(X5), X11 // x[1] + MOV 16(X5), X14 // x[2] + MOV 24(X5), X17 // x[3] + + ADD X8, X29, X10 // z[0] = x[0] + c + SLTU X8, X10, X29 // next c + + ADD X11, X29, X13 // z[1] = x[1] + c + SLTU X11, X13, X29 // next c + + ADD X14, X29, X16 // z[2] = x[2] + c + SLTU X14, X16, X29 // next c + + ADD X17, X29, X19 // z[3] = x[3] + c + SLTU X17, X19, X29 // next c + + MOV X10, 0(X7) // z[0] + MOV X13, 8(X7) // z[1] + MOV X16, 16(X7) // z[2] + MOV X19, 24(X7) // z[3] + + ADD $32, X5 + ADD $32, X7 + SUB $4, X30 + + BGEU X30, X28, loop4 + BEQZ X30, done + +loop1: + MOV 0(X5), X10 // x + + ADD X10, X29, X12 // z = x + c + SLTU X10, X12, X29 // next c + + MOV X12, 0(X7) // z + + ADD $8, X5 + ADD $8, X7 + SUB $1, X30 + + BNEZ X30, loop1 + +done: + MOV X29, c+56(FP) // return c + RET TEXT ·subVW(SB),NOSPLIT,$0 - JMP ·subVW_g(SB) + MOV x+24(FP), X5 + MOV y+48(FP), X6 + MOV z+0(FP), X7 + MOV z_len+8(FP), X30 + + MOV $4, X28 + MOV X6, X29 // b = y + + BEQZ X30, done + BLTU X30, X28, loop1 + +loop4: + MOV 0(X5), X8 // x[0] + MOV 8(X5), X11 // x[1] + MOV 16(X5), X14 // x[2] + MOV 24(X5), X17 // x[3] + + SUB X29, X8, X10 // z[0] = x[0] - b + SLTU X10, X8, X29 // next b + + SUB X29, X11, X13 // z[1] = x[1] - b + SLTU X13, X11, X29 // next b + + SUB X29, X14, X16 // z[2] = x[2] - b + SLTU X16, X14, X29 // next b + + SUB X29, X17, X19 // z[3] = x[3] - b + SLTU X19, X17, X29 // next b + + MOV X10, 0(X7) // z[0] + MOV X13, 8(X7) // z[1] + MOV X16, 16(X7) // z[2] + MOV X19, 24(X7) // z[3] + + ADD $32, X5 + ADD $32, X7 + SUB $4, X30 + + BGEU X30, X28, loop4 + BEQZ X30, done + +loop1: + MOV 0(X5), X10 // x + + SUB X29, X10, X12 // z = x - b + SLTU X12, X10, X29 // next b + + MOV X12, 0(X7) // z + + ADD $8, X5 + ADD $8, X7 + SUB $1, X30 + + BNEZ X30, loop1 + +done: + MOV X29, c+56(FP) // return b + RET TEXT ·shlVU(SB),NOSPLIT,$0 JMP ·shlVU_g(SB) @@ -28,8 +300,171 @@ TEXT ·shrVU(SB),NOSPLIT,$0 JMP ·shrVU_g(SB) TEXT ·mulAddVWW(SB),NOSPLIT,$0 - JMP ·mulAddVWW_g(SB) + MOV x+24(FP), X5 + MOV y+48(FP), X6 + MOV z+0(FP), X7 + MOV z_len+8(FP), X30 + MOV r+56(FP), X29 + + MOV $4, X28 + + BEQ ZERO, X30, done + BLTU X30, X28, loop1 + +loop4: + MOV 0(X5), X8 // x[0] + MOV 8(X5), X11 // x[1] + MOV 16(X5), X14 // x[2] + MOV 24(X5), X17 // x[3] + + MULHU X8, X6, X9 // z_hi[0] = x[0] * y + MUL X8, X6, X8 // z_lo[0] = x[0] * y + ADD X8, X29, X10 // z[0] = z_lo[0] + c + SLTU X8, X10, X23 + ADD X23, X9, X29 // next c + + MULHU X11, X6, X12 // z_hi[1] = x[1] * y + MUL X11, X6, X11 // z_lo[1] = x[1] * y + ADD X11, X29, X13 // z[1] = z_lo[1] + c + SLTU X11, X13, X23 + ADD X23, X12, X29 // next c + + MULHU X14, X6, X15 // z_hi[2] = x[2] * y + MUL X14, X6, X14 // z_lo[2] = x[2] * y + ADD X14, X29, X16 // z[2] = z_lo[2] + c + SLTU X14, X16, X23 + ADD X23, X15, X29 // next c + + MULHU X17, X6, X18 // z_hi[3] = x[3] * y + MUL X17, X6, X17 // z_lo[3] = x[3] * y + ADD X17, X29, X19 // z[3] = z_lo[3] + c + SLTU X17, X19, X23 + ADD X23, X18, X29 // next c + + MOV X10, 0(X7) // z[0] + MOV X13, 8(X7) // z[1] + MOV X16, 16(X7) // z[2] + MOV X19, 24(X7) // z[3] + + ADD $32, X5 + ADD $32, X7 + SUB $4, X30 + + BGEU X30, X28, loop4 + BEQZ X30, done + +loop1: + MOV 0(X5), X10 // x + + MULHU X10, X6, X12 // z_hi = x * y + MUL X10, X6, X10 // z_lo = x * y + ADD X10, X29, X13 // z_lo + c + SLTU X10, X13, X15 + ADD X12, X15, X29 // next c + + MOV X13, 0(X7) // z + + ADD $8, X5 + ADD $8, X7 + SUB $1, X30 + + BNEZ X30, loop1 + +done: + MOV X29, c+64(FP) // return c + RET TEXT ·addMulVVW(SB),NOSPLIT,$0 - JMP ·addMulVVW_g(SB) + MOV x+24(FP), X5 + MOV y+48(FP), X6 + MOV z+0(FP), X7 + MOV z_len+8(FP), X30 + MOV $4, X28 + MOV $0, X29 // c = 0 + + BEQZ X30, done + BLTU X30, X28, loop1 + +loop4: + MOV 0(X5), X8 // x[0] + MOV 0(X7), X10 // z[0] + MOV 8(X5), X11 // x[1] + MOV 8(X7), X13 // z[1] + MOV 16(X5), X14 // x[2] + MOV 16(X7), X16 // z[2] + MOV 24(X5), X17 // x[3] + MOV 24(X7), X19 // z[3] + + MULHU X8, X6, X9 // z_hi[0] = x[0] * y + MUL X8, X6, X8 // z_lo[0] = x[0] * y + ADD X8, X10, X21 // z_lo[0] = x[0] * y + z[0] + SLTU X8, X21, X22 + ADD X9, X22, X9 // z_hi[0] = x[0] * y + z[0] + ADD X21, X29, X10 // z[0] = x[0] * y + z[0] + c + SLTU X21, X10, X22 + ADD X9, X22, X29 // next c + + MULHU X11, X6, X12 // z_hi[1] = x[1] * y + MUL X11, X6, X11 // z_lo[1] = x[1] * y + ADD X11, X13, X21 // z_lo[1] = x[1] * y + z[1] + SLTU X11, X21, X22 + ADD X12, X22, X12 // z_hi[1] = x[1] * y + z[1] + ADD X21, X29, X13 // z[1] = x[1] * y + z[1] + c + SLTU X21, X13, X22 + ADD X12, X22, X29 // next c + + MULHU X14, X6, X15 // z_hi[2] = x[2] * y + MUL X14, X6, X14 // z_lo[2] = x[2] * y + ADD X14, X16, X21 // z_lo[2] = x[2] * y + z[2] + SLTU X14, X21, X22 + ADD X15, X22, X15 // z_hi[2] = x[2] * y + z[2] + ADD X21, X29, X16 // z[2] = x[2] * y + z[2] + c + SLTU X21, X16, X22 + ADD X15, X22, X29 // next c + + MULHU X17, X6, X18 // z_hi[3] = x[3] * y + MUL X17, X6, X17 // z_lo[3] = x[3] * y + ADD X17, X19, X21 // z_lo[3] = x[3] * y + z[3] + SLTU X17, X21, X22 + ADD X18, X22, X18 // z_hi[3] = x[3] * y + z[3] + ADD X21, X29, X19 // z[3] = x[3] * y + z[3] + c + SLTU X21, X19, X22 + ADD X18, X22, X29 // next c + + MOV X10, 0(X7) // z[0] + MOV X13, 8(X7) // z[1] + MOV X16, 16(X7) // z[2] + MOV X19, 24(X7) // z[3] + + ADD $32, X5 + ADD $32, X7 + SUB $4, X30 + + BGEU X30, X28, loop4 + BEQZ X30, done + +loop1: + MOV 0(X5), X10 // x + MOV 0(X7), X11 // z + + MULHU X10, X6, X12 // z_hi = x * y + MUL X10, X6, X10 // z_lo = x * y + ADD X10, X11, X13 // z_lo = x * y + z + SLTU X10, X13, X15 + ADD X12, X15, X12 // z_hi = x * y + z + ADD X13, X29, X10 // z = x * y + z + c + SLTU X13, X10, X15 + ADD X12, X15, X29 // next c + + MOV X10, 0(X7) // z + + ADD $8, X5 + ADD $8, X7 + SUB $1, X30 + + BNEZ X30, loop1 + +done: + MOV X29, c+56(FP) // return c + RET diff --git a/src/math/big/floatmarsh.go b/src/math/big/floatmarsh.go index 16be9469..e220cbc9 100644 --- a/src/math/big/floatmarsh.go +++ b/src/math/big/floatmarsh.go @@ -48,10 +48,10 @@ func (x *Float) GobEncode() ([]byte, error) { b |= 1 } buf[1] = b - byteorder.BePutUint32(buf[2:], x.prec) + byteorder.BEPutUint32(buf[2:], x.prec) if x.form == finite { - byteorder.BePutUint32(buf[6:], uint32(x.exp)) + byteorder.BEPutUint32(buf[6:], uint32(x.exp)) x.mant[len(x.mant)-n:].bytes(buf[10:]) // cut off unused trailing words } @@ -84,13 +84,13 @@ func (z *Float) GobDecode(buf []byte) error { z.acc = Accuracy((b>>3)&3) - 1 z.form = form((b >> 1) & 3) z.neg = b&1 != 0 - z.prec = byteorder.BeUint32(buf[2:]) + z.prec = byteorder.BEUint32(buf[2:]) if z.form == finite { if len(buf) < 10 { return errors.New("Float.GobDecode: buffer too small for finite form float") } - z.exp = int32(byteorder.BeUint32(buf[6:])) + z.exp = int32(byteorder.BEUint32(buf[6:])) z.mant = z.mant.setBytes(buf[10:]) } @@ -106,15 +106,21 @@ func (z *Float) GobDecode(buf []byte) error { return nil } +// AppendText implements the [encoding.TextAppender] interface. +// Only the [Float] value is marshaled (in full precision), other +// attributes such as precision or accuracy are ignored. +func (x *Float) AppendText(b []byte) ([]byte, error) { + if x == nil { + return append(b, ""...), nil + } + return x.Append(b, 'g', -1), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface. // Only the [Float] value is marshaled (in full precision), other // attributes such as precision or accuracy are ignored. func (x *Float) MarshalText() (text []byte, err error) { - if x == nil { - return []byte(""), nil - } - var buf []byte - return x.Append(buf, 'g', -1), nil + return x.AppendText(nil) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/math/big/floatmarsh_test.go b/src/math/big/floatmarsh_test.go index 20def68a..339cb537 100644 --- a/src/math/big/floatmarsh_test.go +++ b/src/math/big/floatmarsh_test.go @@ -171,3 +171,46 @@ func TestFloatGobDecodeInvalid(t *testing.T) { } } } + +func TestFloatAppendText(t *testing.T) { + for _, test := range floatVals { + for _, sign := range []string{"", "+", "-"} { + for _, prec := range []uint{0, 1, 2, 10, 53, 64, 100, 1000} { + if prec > 53 && testing.Short() { + continue + } + x := sign + test + var tx Float + _, _, err := tx.SetPrec(prec).Parse(x, 0) + if err != nil { + t.Errorf("parsing of %s (prec = %d) failed (invalid test case): %v", x, prec, err) + continue + } + buf := make([]byte, 4, 32) + b, err := tx.AppendText(buf) + if err != nil { + t.Errorf("marshaling of %v (prec = %d) failed: %v", &tx, prec, err) + continue + } + var rx Float + rx.SetPrec(prec) + if err := rx.UnmarshalText(b[4:]); err != nil { + t.Errorf("unmarshaling of %v (prec = %d) failed: %v", &tx, prec, err) + continue + } + if rx.Cmp(&tx) != 0 { + t.Errorf("AppendText of %v (prec = %d) failed: got %v want %v", &tx, prec, &rx, &tx) + } + } + } + } +} + +func TestFloatAppendTextNil(t *testing.T) { + var x *Float + buf := make([]byte, 4, 16) + data, _ := x.AppendText(buf) + if string(data[4:]) != "" { + t.Errorf("got %q, want ", data[4:]) + } +} diff --git a/src/math/big/ftoa.go b/src/math/big/ftoa.go index f7a4345d..c5939d73 100644 --- a/src/math/big/ftoa.go +++ b/src/math/big/ftoa.go @@ -309,7 +309,7 @@ func fmtF(buf []byte, prec int, d decimal) []byte { } // fmtB appends the string of x in the format mantissa "p" exponent -// with a decimal mantissa and a binary exponent, or 0" if x is zero, +// with a decimal mantissa and a binary exponent, or "0" if x is zero, // and returns the extended buffer. // The mantissa is normalized such that is uses x.Prec() bits in binary // representation. diff --git a/src/math/big/int.go b/src/math/big/int.go index 944b70c0..df44e9dc 100644 --- a/src/math/big/int.go +++ b/src/math/big/int.go @@ -288,7 +288,7 @@ func (z *Int) Rem(x, y *Int) *Int { // r = x - y*q // // (See Daan Leijen, “Division and Modulus for Computer Scientists”.) -// See [DivMod] for Euclidean division and modulus (unlike Go). +// See [Int.DivMod] for Euclidean division and modulus (unlike Go). func (z *Int) QuoRem(x, y, r *Int) (*Int, *Int) { z.abs, r.abs = z.abs.div(r.abs, x.abs, y.abs) z.neg, r.neg = len(z.abs) > 0 && x.neg != y.neg, len(r.abs) > 0 && x.neg // 0 has no sign diff --git a/src/math/big/intmarsh.go b/src/math/big/intmarsh.go index 56eeefb8..858ca0fa 100644 --- a/src/math/big/intmarsh.go +++ b/src/math/big/intmarsh.go @@ -45,12 +45,14 @@ func (z *Int) GobDecode(buf []byte) error { return nil } +// AppendText implements the [encoding.TextAppender] interface. +func (x *Int) AppendText(b []byte) (text []byte, err error) { + return x.Append(b, 10), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface. func (x *Int) MarshalText() (text []byte, err error) { - if x == nil { - return []byte(""), nil - } - return x.abs.itoa(x.neg, 10), nil + return x.AppendText(nil) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/math/big/intmarsh_test.go b/src/math/big/intmarsh_test.go index 8e7d29f9..681f3dd9 100644 --- a/src/math/big/intmarsh_test.go +++ b/src/math/big/intmarsh_test.go @@ -132,3 +132,36 @@ func TestIntXMLEncoding(t *testing.T) { } } } + +func TestIntAppendText(t *testing.T) { + for _, test := range encodingTests { + for _, sign := range []string{"", "+", "-"} { + x := sign + test + var tx Int + tx.SetString(x, 10) + buf := make([]byte, 4, 32) + b, err := tx.AppendText(buf) + if err != nil { + t.Errorf("marshaling of %s failed: %s", &tx, err) + continue + } + var rx Int + if err := rx.UnmarshalText(b[4:]); err != nil { + t.Errorf("unmarshaling of %s failed: %s", &tx, err) + continue + } + if rx.Cmp(&tx) != 0 { + t.Errorf("AppendText of %s failed: got %s want %s", &tx, &rx, &tx) + } + } + } +} + +func TestIntAppendTextNil(t *testing.T) { + var x *Int + buf := make([]byte, 4, 16) + data, _ := x.AppendText(buf) + if string(data[4:]) != "" { + t.Errorf("got %q, want ", data[4:]) + } +} diff --git a/src/math/big/nat.go b/src/math/big/nat.go index 23b2a0b8..541da229 100644 --- a/src/math/big/nat.go +++ b/src/math/big/nat.go @@ -1321,9 +1321,9 @@ func (z nat) bytes(buf []byte) (i int) { // bigEndianWord returns the contents of buf interpreted as a big-endian encoded Word value. func bigEndianWord(buf []byte) Word { if _W == 64 { - return Word(byteorder.BeUint64(buf)) + return Word(byteorder.BEUint64(buf)) } - return Word(byteorder.BeUint32(buf)) + return Word(byteorder.BEUint32(buf)) } // setBytes interprets buf as the bytes of a big-endian unsigned diff --git a/src/math/big/natdiv.go b/src/math/big/natdiv.go index b55f9990..2e66e342 100644 --- a/src/math/big/natdiv.go +++ b/src/math/big/natdiv.go @@ -602,7 +602,7 @@ func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { v := *vp shlVU(v, vIn, shift) u = u.make(len(uIn) + 1) - u[len(uIn)] = shlVU(u[0:len(uIn)], uIn, shift) + u[len(uIn)] = shlVU(u[:len(uIn)], uIn, shift) // The caller should not pass aliased z and u, since those are // the two different outputs, but correct just in case. @@ -642,15 +642,14 @@ func (q nat) divBasic(u, v nat) { vn1 := v[n-1] rec := reciprocalWord(vn1) + // Invent a leading 0 for u, for the first iteration. + // Invariant: ujn == u[j+n] in each iteration. + ujn := Word(0) + // Compute each digit of quotient. for j := m; j >= 0; j-- { // Compute the 2-by-1 guess q̂. - // The first iteration must invent a leading 0 for u. qhat := Word(_M) - var ujn Word - if j+n < len(u) { - ujn = u[j+n] - } // ujn ≤ vn1, or else q̂ would be more than one digit. // For ujn == vn1, we set q̂ to the max digit M above. @@ -699,6 +698,8 @@ func (q nat) divBasic(u, v nat) { qhat-- } + ujn = u[j+n-1] + // Save quotient digit. // Caller may know the top digit is zero and not leave room for it. if j == m && m == len(q) && qhat == 0 { @@ -884,7 +885,7 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { if qhatv.cmp(u.norm()) > 0 { panic("impossible") } - c := subVV(u[0:len(qhatv)], u[0:len(qhatv)], qhatv) + c := subVV(u[:len(qhatv)], u[:len(qhatv)], qhatv) if c > 0 { c = subVW(u[len(qhatv):], u[len(qhatv):], c) } diff --git a/src/math/big/ratconv.go b/src/math/big/ratconv.go index dd99aecd..12f9888c 100644 --- a/src/math/big/ratconv.go +++ b/src/math/big/ratconv.go @@ -299,12 +299,13 @@ func scanExponent(r io.ByteScanner, base2ok, sepOk bool) (exp int64, base int, e // String returns a string representation of x in the form "a/b" (even if b == 1). func (x *Rat) String() string { - return string(x.marshal()) + return string(x.marshal(nil)) } -// marshal implements String returning a slice of bytes -func (x *Rat) marshal() []byte { - var buf []byte +// marshal implements [Rat.String] returning a slice of bytes. +// It appends the string representation of x in the form "a/b" (even if b == 1) to buf, +// and returns the extended buffer. +func (x *Rat) marshal(buf []byte) []byte { buf = x.a.Append(buf, 10) buf = append(buf, '/') if len(x.b.abs) != 0 { diff --git a/src/math/big/ratmarsh.go b/src/math/big/ratmarsh.go index 69628294..26ded1d9 100644 --- a/src/math/big/ratmarsh.go +++ b/src/math/big/ratmarsh.go @@ -29,7 +29,7 @@ func (x *Rat) GobEncode() ([]byte, error) { // this should never happen return nil, errors.New("Rat.GobEncode: numerator too large") } - byteorder.BePutUint32(buf[j-4:j], uint32(n)) + byteorder.BEPutUint32(buf[j-4:j], uint32(n)) j -= 1 + 4 b := ratGobVersion << 1 // make space for sign bit if x.a.neg { @@ -54,7 +54,7 @@ func (z *Rat) GobDecode(buf []byte) error { return fmt.Errorf("Rat.GobDecode: encoding version %d not supported", b>>1) } const j = 1 + 4 - ln := byteorder.BeUint32(buf[j-4 : j]) + ln := byteorder.BEUint32(buf[j-4 : j]) if uint64(ln) > math.MaxInt-j { return errors.New("Rat.GobDecode: invalid length") } @@ -68,12 +68,17 @@ func (z *Rat) GobDecode(buf []byte) error { return nil } +// AppendText implements the [encoding.TextAppender] interface. +func (x *Rat) AppendText(b []byte) ([]byte, error) { + if x.IsInt() { + return x.a.AppendText(b) + } + return x.marshal(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface. func (x *Rat) MarshalText() (text []byte, err error) { - if x.IsInt() { - return x.a.MarshalText() - } - return x.marshal(), nil + return x.AppendText(nil) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/math/big/ratmarsh_test.go b/src/math/big/ratmarsh_test.go index 15c933ef..7d139bca 100644 --- a/src/math/big/ratmarsh_test.go +++ b/src/math/big/ratmarsh_test.go @@ -136,3 +136,26 @@ func TestRatGobDecodeShortBuffer(t *testing.T) { } } } + +func TestRatAppendText(t *testing.T) { + for _, num := range ratNums { + for _, denom := range ratDenoms { + var tx Rat + tx.SetString(num + "/" + denom) + buf := make([]byte, 4, 32) + b, err := tx.AppendText(buf) + if err != nil { + t.Errorf("marshaling of %s failed: %s", &tx, err) + continue + } + var rx Rat + if err := rx.UnmarshalText(b[4:]); err != nil { + t.Errorf("unmarshaling of %s failed: %s", &tx, err) + continue + } + if rx.Cmp(&tx) != 0 { + t.Errorf("AppendText of %s failed: got %s want %s", &tx, &rx, &tx) + } + } + } +} diff --git a/src/math/bits/bits.go b/src/math/bits/bits.go index 235d63e8..76ed1d03 100644 --- a/src/math/bits/bits.go +++ b/src/math/bits/bits.go @@ -38,7 +38,7 @@ func LeadingZeros64(x uint64) int { return 64 - Len64(x) } // --- TrailingZeros --- -// See http://supertech.csail.mit.edu/papers/debruijn.pdf +// See http://keithandkatie.com/keith/papers/debruijn.html const deBruijn32 = 0x077CB531 var deBruijn32tab = [32]byte{ diff --git a/src/math/bits/bits_test.go b/src/math/bits/bits_test.go index 23b4539f..6f6e1c22 100644 --- a/src/math/bits/bits_test.go +++ b/src/math/bits/bits_test.go @@ -1109,7 +1109,7 @@ func TestDiv64PanicZero(t *testing.T) { } func TestRem32(t *testing.T) { - // Sanity check: for non-oveflowing dividends, the result is the + // Sanity check: for non-overflowing dividends, the result is the // same as the rem returned by Div32 hi, lo, y := uint32(510510), uint32(9699690), uint32(510510+1) // ensure hi < y for i := 0; i < 1000; i++ { @@ -1136,7 +1136,7 @@ func TestRem32Overflow(t *testing.T) { } func TestRem64(t *testing.T) { - // Sanity check: for non-oveflowing dividends, the result is the + // Sanity check: for non-overflowing dividends, the result is the // same as the rem returned by Div64 hi, lo, y := uint64(510510), uint64(9699690), uint64(510510+1) // ensure hi < y for i := 0; i < 1000; i++ { diff --git a/src/math/dim_asm.go b/src/math/dim_asm.go index f4adbd0a..a1d23dd0 100644 --- a/src/math/dim_asm.go +++ b/src/math/dim_asm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 || arm64 || riscv64 || s390x +//go:build amd64 || arm64 || loong64 || riscv64 || s390x package math diff --git a/src/math/dim_loong64.s b/src/math/dim_loong64.s new file mode 100644 index 00000000..1484bf76 --- /dev/null +++ b/src/math/dim_loong64.s @@ -0,0 +1,77 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +#define PosInf 0x7FF0000000000000 +#define NaN 0x7FF8000000000001 +#define NegInf 0xFFF0000000000000 + +TEXT ·archMax(SB),NOSPLIT,$0 + MOVD x+0(FP), F0 + MOVD y+8(FP), F1 + FCLASSD F0, F2 + FCLASSD F1, F3 + + // combine x and y categories together to judge + MOVV F2, R4 + MOVV F3, R5 + OR R5, R4 + + // +Inf special cases + AND $64, R4, R5 + BNE R5, isPosInf + + // NaN special cases + AND $2, R4, R5 + BNE R5, isMaxNaN + + // normal case + FMAXD F0, F1, F0 + MOVD F0, ret+16(FP) + RET + +isMaxNaN: + MOVV $NaN, R6 + MOVV R6, ret+16(FP) + RET + +isPosInf: + MOVV $PosInf, R6 + MOVV R6, ret+16(FP) + RET + +TEXT ·archMin(SB),NOSPLIT,$0 + MOVD x+0(FP), F0 + MOVD y+8(FP), F1 + FCLASSD F0, F2 + FCLASSD F1, F3 + + // combine x and y categories together to judge + MOVV F2, R4 + MOVV F3, R5 + OR R5, R4 + + // -Inf special cases + AND $4, R4, R5 + BNE R5, isNegInf + + // NaN special cases + AND $2, R4, R5 + BNE R5, isMinNaN + + // normal case + FMIND F0, F1, F0 + MOVD F0, ret+16(FP) + RET + +isMinNaN: + MOVV $NaN, R6 + MOVV R6, ret+16(FP) + RET + +isNegInf: + MOVV $NegInf, R6 + MOVV R6, ret+16(FP) + RET diff --git a/src/math/dim_noasm.go b/src/math/dim_noasm.go index 5b9e06fe..6f4917b8 100644 --- a/src/math/dim_noasm.go +++ b/src/math/dim_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm64 && !riscv64 && !s390x +//go:build !amd64 && !arm64 && !loong64 && !riscv64 && !s390x package math diff --git a/src/math/floor_asm.go b/src/math/floor_asm.go index fb419d6d..1b06b8de 100644 --- a/src/math/floor_asm.go +++ b/src/math/floor_asm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build 386 || amd64 || arm64 || ppc64 || ppc64le || s390x || wasm +//go:build 386 || amd64 || arm64 || loong64 || ppc64 || ppc64le || riscv64 || s390x || wasm package math diff --git a/src/math/floor_loong64.s b/src/math/floor_loong64.s new file mode 100644 index 00000000..0df7deee --- /dev/null +++ b/src/math/floor_loong64.s @@ -0,0 +1,41 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// derived from math/floor_riscv64.s + +#include "textflag.h" + +#define ROUNDFN(NAME, FUNC) \ +TEXT NAME(SB),NOSPLIT,$0; \ + MOVD x+0(FP), F0; \ + MOVV F0, R11; \ + /* 1023: bias of exponent, [-2^53, 2^53]: exactly integer represent range */; \ + MOVV $1023+53, R12; \ + /* Drop all fraction bits */; \ + SRLV $52, R11, R11; \ + /* Remove sign bit */; \ + AND $0x7FF, R11, R11; \ + BLTU R12, R11, isExtremum; \ +normal:; \ + FUNC F0, F2; \ + MOVV F2, R10; \ + BEQ R10, R0, is0; \ + FFINTDV F2, F0; \ +/* Return either input is +-Inf, NaN(0x7FF) or out of precision limitation */; \ +isExtremum:; \ + MOVD F0, ret+8(FP); \ + RET; \ +is0:; \ + FCOPYSGD F0, F2, F2; \ + MOVD F2, ret+8(FP); \ + RET + +// func archFloor(x float64) float64 +ROUNDFN(·archFloor, FTINTRMVD) + +// func archCeil(x float64) float64 +ROUNDFN(·archCeil, FTINTRPVD) + +// func archTrunc(x float64) float64 +ROUNDFN(·archTrunc, FTINTRZVD) diff --git a/src/math/floor_noasm.go b/src/math/floor_noasm.go index 5641c7ea..34bd292f 100644 --- a/src/math/floor_noasm.go +++ b/src/math/floor_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !386 && !amd64 && !arm64 && !ppc64 && !ppc64le && !s390x && !wasm +//go:build !386 && !amd64 && !arm64 && !loong64 && !ppc64 && !ppc64le && !riscv64 && !s390x && !wasm package math diff --git a/src/math/floor_riscv64.s b/src/math/floor_riscv64.s new file mode 100644 index 00000000..d9fe0ed8 --- /dev/null +++ b/src/math/floor_riscv64.s @@ -0,0 +1,48 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// RISC-V offered floating-point (FP) rounding by FP conversion instructions (FCVT) +// with rounding mode field. +// As Go spec expects FP rounding result in FP, we have to use FCVT integer +// back to FP (fp -> int -> fp). +// RISC-V only set Inexact flag during invalid FP-integer conversion without changing any data, +// on the other hand, RISC-V sets out of integer represent range yet valid FP into NaN. +// When it comes to integer-FP conversion, invalid FP like NaN, +-Inf will be +// converted into the closest valid FP, for example: +// +// `Floor(-Inf) -> int64(0x7fffffffffffffff) -> float64(9.22e+18)` +// `Floor(18446744073709549568.0) -> int64(0x7fffffffffffffff) -> float64(9.22e+18)` +// +// This ISA conversion limitation requires we skip all invalid or out of range FP +// before any normal rounding operations. + +#define ROUNDFN(NAME, MODE) \ +TEXT NAME(SB),NOSPLIT,$0; \ + MOVD x+0(FP), F10; \ + FMVXD F10, X10; \ + /* Drop all fraction bits */;\ + SRL $52, X10, X12; \ + /* Remove sign bit */; \ + AND $0x7FF, X12, X12;\ + /* Return either input is +-Inf, NaN(0x7FF) or out of precision limitation */;\ + /* 1023: bias of exponent, [-2^53, 2^53]: exactly integer represent range */;\ + MOV $1023+53, X11; \ + BLTU X11, X12, 4(PC);\ + FCVTLD.MODE F10, X11; \ + FCVTDL X11, F11; \ + /* RISC-V rounds negative values to +0, restore original sign */;\ + FSGNJD F10, F11, F10; \ + MOVD F10, ret+8(FP); \ + RET + +// func archFloor(x float64) float64 +ROUNDFN(·archFloor, RDN) + +// func archCeil(x float64) float64 +ROUNDFN(·archCeil, RUP) + +// func archTrunc(x float64) float64 +ROUNDFN(·archTrunc, RTZ) diff --git a/src/math/rand/default_test.go b/src/math/rand/default_test.go index 19fd75df..b42cd165 100644 --- a/src/math/rand/default_test.go +++ b/src/math/rand/default_test.go @@ -37,11 +37,7 @@ func TestDefaultRace(t *testing.T) { i := i t.Run(strconv.Itoa(i), func(t *testing.T) { t.Parallel() - exe, err := os.Executable() - if err != nil { - exe = os.Args[0] - } - cmd := testenv.Command(t, exe, "-test.run=TestDefaultRace") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=TestDefaultRace") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, fmt.Sprintf("GO_RAND_TEST_HELPER_CODE=%d", i/2)) if i%2 != 0 { diff --git a/src/math/rand/rand.go b/src/math/rand/rand.go index 61ff5c1b..4be1ca20 100644 --- a/src/math/rand/rand.go +++ b/src/math/rand/rand.go @@ -313,6 +313,9 @@ var globalRandGenerator atomic.Pointer[Rand] var randautoseed = godebug.New("randautoseed") +// randseednop controls whether the global Seed is a no-op. +var randseednop = godebug.New("randseednop") + // globalRand returns the generator to use for the top-level convenience // functions. func globalRand() *Rand { @@ -391,7 +394,15 @@ func (fs *runtimeSource) read(p []byte, readVal *int64, readPos *int8) (n int, e // a random value. Programs that call Seed with a known value to get // a specific sequence of results should use New(NewSource(seed)) to // obtain a local random generator. +// +// As of Go 1.24 [Seed] is a no-op. To restore the previous behavior set +// GODEBUG=randseednop=0. func Seed(seed int64) { + if randseednop.Value() != "0" { + return + } + randseednop.IncNonDefault() + orig := globalRandGenerator.Load() // If we are already using a lockedSource, we can just re-seed it. diff --git a/src/math/rand/rand_test.go b/src/math/rand/rand_test.go index 7906f296..1e1fad79 100644 --- a/src/math/rand/rand_test.go +++ b/src/math/rand/rand_test.go @@ -556,6 +556,42 @@ func TestUniformFactorial(t *testing.T) { } } +func TestSeedNop(t *testing.T) { + // If the global Seed takes effect, then resetting it to a certain value + // should provide predictable output to functions using it. + t.Run("randseednop=0", func(t *testing.T) { + t.Setenv("GODEBUG", "randseednop=0") + Seed(1) + before := Int63() + Seed(1) + after := Int63() + if before != after { + t.Fatal("global Seed should take effect") + } + }) + // If calls to the global Seed are no-op then functions using it should + // provide different output, even if it was reset to the same value. + t.Run("randseednop=1", func(t *testing.T) { + t.Setenv("GODEBUG", "randseednop=1") + Seed(1) + before := Int63() + Seed(1) + after := Int63() + if before == after { + t.Fatal("global Seed should be a no-op") + } + }) + t.Run("GODEBUG unset", func(t *testing.T) { + Seed(1) + before := Int63() + Seed(1) + after := Int63() + if before == after { + t.Fatal("global Seed should default to being a no-op") + } + }) +} + // Benchmarks func BenchmarkInt63Threadsafe(b *testing.B) { diff --git a/src/math/rand/v2/chacha8.go b/src/math/rand/v2/chacha8.go index f9eaacf6..eb43e659 100644 --- a/src/math/rand/v2/chacha8.go +++ b/src/math/rand/v2/chacha8.go @@ -58,19 +58,19 @@ func (c *ChaCha8) Read(p []byte) (n int, err error) { p = p[n:] } for len(p) >= 8 { - byteorder.LePutUint64(p, c.Uint64()) + byteorder.LEPutUint64(p, c.Uint64()) p = p[8:] n += 8 } if len(p) > 0 { - byteorder.LePutUint64(c.readBuf[:], c.Uint64()) + byteorder.LEPutUint64(c.readBuf[:], c.Uint64()) n += copy(p, c.readBuf[:]) c.readLen = 8 - len(p) } return } -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. func (c *ChaCha8) UnmarshalBinary(data []byte) error { data, ok := cutPrefix(data, []byte("readbuf:")) if ok { @@ -98,13 +98,18 @@ func readUint8LengthPrefixed(b []byte) (buf, rest []byte, ok bool) { return b[1 : 1+b[0]], b[1+b[0]:], true } -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (c *ChaCha8) MarshalBinary() ([]byte, error) { +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (c *ChaCha8) AppendBinary(b []byte) ([]byte, error) { if c.readLen > 0 { - out := []byte("readbuf:") - out = append(out, uint8(c.readLen)) - out = append(out, c.readBuf[len(c.readBuf)-c.readLen:]...) - return append(out, chacha8rand.Marshal(&c.state)...), nil + b = append(b, "readbuf:"...) + b = append(b, uint8(c.readLen)) + b = append(b, c.readBuf[len(c.readBuf)-c.readLen:]...) } - return chacha8rand.Marshal(&c.state), nil + return append(b, chacha8rand.Marshal(&c.state)...), nil +} + +// MarshalBinary implements the [encoding.BinaryMarshaler] interface. +func (c *ChaCha8) MarshalBinary() ([]byte, error) { + // the maximum length of (chacha8rand.Marshal + c.readBuf + "readbuf:") is 64 + return c.AppendBinary(make([]byte, 0, 64)) } diff --git a/src/math/rand/v2/chacha8_test.go b/src/math/rand/v2/chacha8_test.go index 50e83ea1..ba11b7cc 100644 --- a/src/math/rand/v2/chacha8_test.go +++ b/src/math/rand/v2/chacha8_test.go @@ -98,6 +98,22 @@ func TestChaCha8Read(t *testing.T) { } } +func BenchmarkChaCha8MarshalBinary(b *testing.B) { + p := NewChaCha8(chacha8seed) + for range b.N { + p.MarshalBinary() + } +} + +func BenchmarkChaCha8MarshalBinaryRead(b *testing.B) { + p := NewChaCha8(chacha8seed) + buf := make([]byte, 1) + for range b.N { + p.MarshalBinary() + p.Read(buf) + } +} + func TestChaCha8Marshal(t *testing.T) { p := NewChaCha8(chacha8seed) for i, x := range chacha8output { @@ -108,6 +124,17 @@ func TestChaCha8Marshal(t *testing.T) { if string(enc) != chacha8marshal[i] { t.Errorf("#%d: MarshalBinary=%q, want %q", i, enc, chacha8marshal[i]) } + + b := make([]byte, 4, 32) + b, err = p.AppendBinary(b) + encAppend := b[4:] + if err != nil { + t.Fatalf("#%d: AppendBinary: %v", i, err) + } + if string(encAppend) != chacha8marshal[i] { + t.Errorf("#%d: AppendBinary=%q, want %q", i, encAppend, chacha8marshal[i]) + } + *p = ChaCha8{} if err := p.UnmarshalBinary(enc); err != nil { t.Fatalf("#%d: UnmarshalBinary: %v", i, err) @@ -128,6 +155,17 @@ func TestChaCha8MarshalRead(t *testing.T) { if string(enc) != chacha8marshalread[i] { t.Errorf("#%d: MarshalBinary=%q, want %q", i, enc, chacha8marshalread[i]) } + + b := make([]byte, 4, 32) + b, err = p.AppendBinary(b) + encAppend := b[4:] + if err != nil { + t.Fatalf("#%d: AppendBinary: %v", i, err) + } + if string(encAppend) != chacha8marshalread[i] { + t.Errorf("#%d: AppendBinary=%q, want %q", i, encAppend, chacha8marshalread[i]) + } + *p = ChaCha8{} if err := p.UnmarshalBinary(enc); err != nil { t.Fatalf("#%d: UnmarshalBinary: %v", i, err) diff --git a/src/math/rand/v2/pcg.go b/src/math/rand/v2/pcg.go index 4ccd5e32..9cd7d11a 100644 --- a/src/math/rand/v2/pcg.go +++ b/src/math/rand/v2/pcg.go @@ -31,24 +31,28 @@ func (p *PCG) Seed(seed1, seed2 uint64) { p.lo = seed2 } -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (p *PCG) MarshalBinary() ([]byte, error) { - b := make([]byte, 20) - copy(b, "pcg:") - byteorder.BePutUint64(b[4:], p.hi) - byteorder.BePutUint64(b[4+8:], p.lo) +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (p *PCG) AppendBinary(b []byte) ([]byte, error) { + b = append(b, "pcg:"...) + b = byteorder.BEAppendUint64(b, p.hi) + b = byteorder.BEAppendUint64(b, p.lo) return b, nil } +// MarshalBinary implements the [encoding.BinaryMarshaler] interface. +func (p *PCG) MarshalBinary() ([]byte, error) { + return p.AppendBinary(make([]byte, 0, 20)) +} + var errUnmarshalPCG = errors.New("invalid PCG encoding") -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. func (p *PCG) UnmarshalBinary(data []byte) error { if len(data) != 20 || string(data[:4]) != "pcg:" { return errUnmarshalPCG } - p.hi = byteorder.BeUint64(data[4:]) - p.lo = byteorder.BeUint64(data[4+8:]) + p.hi = byteorder.BEUint64(data[4:]) + p.lo = byteorder.BEUint64(data[4+8:]) return nil } diff --git a/src/math/rand/v2/pcg_test.go b/src/math/rand/v2/pcg_test.go index db866c8c..6558dab7 100644 --- a/src/math/rand/v2/pcg_test.go +++ b/src/math/rand/v2/pcg_test.go @@ -21,9 +21,10 @@ func BenchmarkPCG_DXSM(b *testing.B) { func TestPCGMarshal(t *testing.T) { var p PCG const ( - seed1 = 0x123456789abcdef0 - seed2 = 0xfedcba9876543210 - want = "pcg:\x12\x34\x56\x78\x9a\xbc\xde\xf0\xfe\xdc\xba\x98\x76\x54\x32\x10" + seed1 = 0x123456789abcdef0 + seed2 = 0xfedcba9876543210 + want = "pcg:\x12\x34\x56\x78\x9a\xbc\xde\xf0\xfe\xdc\xba\x98\x76\x54\x32\x10" + wantAppend = "\x00\x00\x00\x00" + want ) p.Seed(seed1, seed2) data, err := p.MarshalBinary() @@ -31,6 +32,12 @@ func TestPCGMarshal(t *testing.T) { t.Errorf("MarshalBinary() = %q, %v, want %q, nil", data, err, want) } + dataAppend := make([]byte, 4, 32) + dataAppend, err = p.AppendBinary(dataAppend) + if string(dataAppend) != wantAppend || err != nil { + t.Errorf("AppendBinary() = %q, %v, want %q, nil", dataAppend, err, wantAppend) + } + q := PCG{} if err := q.UnmarshalBinary([]byte(want)); err != nil { t.Fatalf("UnmarshalBinary(): %v", err) diff --git a/src/math/rand/v2/rand.go b/src/math/rand/v2/rand.go index fea1e3a2..f6b2e475 100644 --- a/src/math/rand/v2/rand.go +++ b/src/math/rand/v2/rand.go @@ -275,12 +275,12 @@ func Uint32() uint32 { return globalRand.Uint32() } // Uint64N returns, as a uint64, a pseudo-random number in the half-open interval [0,n) // from the default Source. -// It panics if n <= 0. +// It panics if n == 0. func Uint64N(n uint64) uint64 { return globalRand.Uint64N(n) } // Uint32N returns, as a uint32, a pseudo-random number in the half-open interval [0,n) // from the default Source. -// It panics if n <= 0. +// It panics if n == 0. func Uint32N(n uint32) uint32 { return globalRand.Uint32N(n) } // Uint64 returns a pseudo-random 64-bit value as a uint64 @@ -314,7 +314,7 @@ func IntN(n int) int { return globalRand.IntN(n) } // UintN returns, as a uint, a pseudo-random number in the half-open interval [0,n) // from the default Source. -// It panics if n <= 0. +// It panics if n == 0. func UintN(n uint) uint { return globalRand.UintN(n) } // N returns a pseudo-random number in the half-open interval [0,n) from the default Source. diff --git a/src/mime/encodedword.go b/src/mime/encodedword.go index 856433f8..c4afad04 100644 --- a/src/mime/encodedword.go +++ b/src/mime/encodedword.go @@ -226,7 +226,7 @@ func (d *WordDecoder) Decode(word string) (string, error) { } // DecodeHeader decodes all encoded-words of the given string. It returns an -// error if and only if WordDecoder.CharsetReader of d returns an error. +// error if and only if [WordDecoder.CharsetReader] of d returns an error. func (d *WordDecoder) DecodeHeader(header string) (string, error) { // If there is no encoded-word, returns before creating a buffer. i := strings.Index(header, "=?") diff --git a/src/mime/mediatype.go b/src/mime/mediatype.go index 97f3563a..f0a0be21 100644 --- a/src/mime/mediatype.go +++ b/src/mime/mediatype.go @@ -7,6 +7,7 @@ package mime import ( "errors" "fmt" + "maps" "slices" "strings" "unicode" @@ -33,13 +34,7 @@ func FormatMediaType(t string, param map[string]string) string { b.WriteString(strings.ToLower(sub)) } - attrs := make([]string, 0, len(param)) - for a := range param { - attrs = append(attrs, a) - } - slices.Sort(attrs) - - for _, attribute := range attrs { + for _, attribute := range slices.Sorted(maps.Keys(param)) { value := param[attribute] b.WriteByte(';') b.WriteByte(' ') diff --git a/src/mime/mediatype_test.go b/src/mime/mediatype_test.go index 1458cdb6..1731f736 100644 --- a/src/mime/mediatype_test.go +++ b/src/mime/mediatype_test.go @@ -5,7 +5,7 @@ package mime import ( - "reflect" + "maps" "strings" "testing" ) @@ -429,7 +429,7 @@ func TestParseMediaType(t *testing.T) { if len(params) == 0 && len(test.p) == 0 { continue } - if !reflect.DeepEqual(params, test.p) { + if !maps.Equal(params, test.p) { t.Errorf("for input %#q, wrong params.\n"+ "expected: %#v\n"+ " got: %#v", diff --git a/src/mime/multipart/writer.go b/src/mime/multipart/writer.go index 5e589c49..818970d7 100644 --- a/src/mime/multipart/writer.go +++ b/src/mime/multipart/writer.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "maps" "net/textproto" "slices" "strings" @@ -107,12 +108,7 @@ func (w *Writer) CreatePart(header textproto.MIMEHeader) (io.Writer, error) { fmt.Fprintf(&b, "--%s\r\n", w.boundary) } - keys := make([]string, 0, len(header)) - for k := range header { - keys = append(keys, k) - } - slices.Sort(keys) - for _, k := range keys { + for _, k := range slices.Sorted(maps.Keys(header)) { for _, v := range header[k] { fmt.Fprintf(&b, "%s: %s\r\n", k, v) } diff --git a/src/mime/type_test.go b/src/mime/type_test.go index d8368e88..6bdf37b6 100644 --- a/src/mime/type_test.go +++ b/src/mime/type_test.go @@ -5,7 +5,8 @@ package mime import ( - "reflect" + "internal/asan" + "slices" "strings" "sync" "testing" @@ -136,13 +137,16 @@ func TestExtensionsByType(t *testing.T) { t.Errorf("ExtensionsByType(%q) = %q, %v; want error substring %q", tt.typ, got, err, tt.wantErr) continue } - if !reflect.DeepEqual(got, tt.want) { + if !slices.Equal(got, tt.want) { t.Errorf("ExtensionsByType(%q) = %q; want %q", tt.typ, got, tt.want) } } } func TestLookupMallocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } n := testing.AllocsPerRun(10000, func() { TypeByExtension(".html") TypeByExtension(".HtML") @@ -213,7 +217,7 @@ func TestExtensionsByType2(t *testing.T) { t.Errorf("ExtensionsByType(%q): %v", tt.typ, err) continue } - if !reflect.DeepEqual(got, tt.want) { + if !slices.Equal(got, tt.want) { t.Errorf("ExtensionsByType(%q) = %q; want %q", tt.typ, got, tt.want) } } diff --git a/src/net/addrselect.go b/src/net/addrselect.go index caff09b3..0ff8ec37 100644 --- a/src/net/addrselect.go +++ b/src/net/addrselect.go @@ -8,7 +8,7 @@ package net import ( "net/netip" - "sort" + "slices" ) func sortByRFC6724(addrs []IPAddr) { @@ -22,19 +22,20 @@ func sortByRFC6724withSrcs(addrs []IPAddr, srcs []netip.Addr) { if len(addrs) != len(srcs) { panic("internal error") } - addrAttr := make([]ipAttr, len(addrs)) - srcAttr := make([]ipAttr, len(srcs)) + addrInfos := make([]byRFC6724Info, len(addrs)) for i, v := range addrs { addrAttrIP, _ := netip.AddrFromSlice(v.IP) - addrAttr[i] = ipAttrOf(addrAttrIP) - srcAttr[i] = ipAttrOf(srcs[i]) + addrInfos[i] = byRFC6724Info{ + addr: addrs[i], + addrAttr: ipAttrOf(addrAttrIP), + src: srcs[i], + srcAttr: ipAttrOf(srcs[i]), + } + } + slices.SortStableFunc(addrInfos, compareByRFC6724) + for i := range addrInfos { + addrs[i] = addrInfos[i].addr } - sort.Stable(&byRFC6724{ - addrs: addrs, - addrAttr: addrAttr, - srcs: srcs, - srcAttr: srcAttr, - }) } // srcAddrs tries to UDP-connect to each address to see if it has a @@ -75,45 +76,36 @@ func ipAttrOf(ip netip.Addr) ipAttr { } } -type byRFC6724 struct { - addrs []IPAddr // addrs to sort - addrAttr []ipAttr - srcs []netip.Addr // or not valid addr if unreachable - srcAttr []ipAttr +type byRFC6724Info struct { + addr IPAddr + addrAttr ipAttr + src netip.Addr + srcAttr ipAttr } -func (s *byRFC6724) Len() int { return len(s.addrs) } +// compareByRFC6724 compares two byRFC6724Info records and returns an integer +// indicating the order. It follows the algorithm and variable names from +// RFC 6724 section 6. Returns -1 if a is preferred, 1 if b is preferred, +// and 0 if they are equal. +func compareByRFC6724(a, b byRFC6724Info) int { + DA := a.addr.IP + DB := b.addr.IP + SourceDA := a.src + SourceDB := b.src + attrDA := &a.addrAttr + attrDB := &b.addrAttr + attrSourceDA := &a.srcAttr + attrSourceDB := &b.srcAttr -func (s *byRFC6724) Swap(i, j int) { - s.addrs[i], s.addrs[j] = s.addrs[j], s.addrs[i] - s.srcs[i], s.srcs[j] = s.srcs[j], s.srcs[i] - s.addrAttr[i], s.addrAttr[j] = s.addrAttr[j], s.addrAttr[i] - s.srcAttr[i], s.srcAttr[j] = s.srcAttr[j], s.srcAttr[i] -} - -// Less reports whether i is a better destination address for this -// host than j. -// -// The algorithm and variable names comes from RFC 6724 section 6. -func (s *byRFC6724) Less(i, j int) bool { - DA := s.addrs[i].IP - DB := s.addrs[j].IP - SourceDA := s.srcs[i] - SourceDB := s.srcs[j] - attrDA := &s.addrAttr[i] - attrDB := &s.addrAttr[j] - attrSourceDA := &s.srcAttr[i] - attrSourceDB := &s.srcAttr[j] - - const preferDA = true - const preferDB = false + const preferDA = -1 + const preferDB = 1 // Rule 1: Avoid unusable destinations. // If DB is known to be unreachable or if Source(DB) is undefined, then // prefer DA. Similarly, if DA is known to be unreachable or if // Source(DA) is undefined, then prefer DB. if !SourceDA.IsValid() && !SourceDB.IsValid() { - return false // "equal" + return 0 // "equal" } if !SourceDB.IsValid() { return preferDA @@ -212,7 +204,7 @@ func (s *byRFC6724) Less(i, j int) bool { // Rule 10: Otherwise, leave the order unchanged. // If DA preceded DB in the original list, prefer DA. // Otherwise, prefer DB. - return false // "equal" + return 0 // "equal" } type policyTableEntry struct { diff --git a/src/net/cgo_solaris.go b/src/net/cgo_solaris.go index cde9c957..98a0e819 100644 --- a/src/net/cgo_solaris.go +++ b/src/net/cgo_solaris.go @@ -7,7 +7,7 @@ package net /* -#cgo LDFLAGS: -lsocket -lnsl -lsendfile +#cgo LDFLAGS: -lsocket -lnsl #include */ import "C" diff --git a/src/net/cgo_unix.go b/src/net/cgo_unix.go index bc374c2c..e866150f 100644 --- a/src/net/cgo_unix.go +++ b/src/net/cgo_unix.go @@ -16,6 +16,7 @@ import ( "errors" "internal/bytealg" "net/netip" + "runtime" "syscall" "unsafe" @@ -195,6 +196,16 @@ func cgoLookupHostIP(network, name string) (addrs []IPAddr, err error) { return nil, newDNSError(err, name, "") case _C_EAI_NONAME, _C_EAI_NODATA: return nil, newDNSError(errNoSuchHost, name, "") + case _C_EAI_ADDRFAMILY: + if runtime.GOOS == "freebsd" { + // FreeBSD began returning EAI_ADDRFAMILY for valid hosts without + // an A record in 13.2. We previously returned "no such host" for + // this case. + // + // https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=273912 + return nil, newDNSError(errNoSuchHost, name, "") + } + fallthrough default: return nil, newDNSError(addrinfoErrno(gerrno), name, "") } diff --git a/src/net/cgo_unix_cgo.go b/src/net/cgo_unix_cgo.go index d38ae0a8..c4b8197c 100644 --- a/src/net/cgo_unix_cgo.go +++ b/src/net/cgo_unix_cgo.go @@ -22,6 +22,11 @@ package net #define EAI_NODATA -5 #endif +// If nothing else defined EAI_ADDRFAMILY, make sure it has a value. +#ifndef EAI_ADDRFAMILY +#define EAI_ADDRFAMILY -9 +#endif + // If nothing else defined EAI_OVERFLOW, make sure it has a value. #ifndef EAI_OVERFLOW #define EAI_OVERFLOW -12 @@ -31,19 +36,20 @@ import "C" import "unsafe" const ( - _C_AF_INET = C.AF_INET - _C_AF_INET6 = C.AF_INET6 - _C_AF_UNSPEC = C.AF_UNSPEC - _C_EAI_AGAIN = C.EAI_AGAIN - _C_EAI_NODATA = C.EAI_NODATA - _C_EAI_NONAME = C.EAI_NONAME - _C_EAI_SERVICE = C.EAI_SERVICE - _C_EAI_OVERFLOW = C.EAI_OVERFLOW - _C_EAI_SYSTEM = C.EAI_SYSTEM - _C_IPPROTO_TCP = C.IPPROTO_TCP - _C_IPPROTO_UDP = C.IPPROTO_UDP - _C_SOCK_DGRAM = C.SOCK_DGRAM - _C_SOCK_STREAM = C.SOCK_STREAM + _C_AF_INET = C.AF_INET + _C_AF_INET6 = C.AF_INET6 + _C_AF_UNSPEC = C.AF_UNSPEC + _C_EAI_ADDRFAMILY = C.EAI_ADDRFAMILY + _C_EAI_AGAIN = C.EAI_AGAIN + _C_EAI_NODATA = C.EAI_NODATA + _C_EAI_NONAME = C.EAI_NONAME + _C_EAI_SERVICE = C.EAI_SERVICE + _C_EAI_OVERFLOW = C.EAI_OVERFLOW + _C_EAI_SYSTEM = C.EAI_SYSTEM + _C_IPPROTO_TCP = C.IPPROTO_TCP + _C_IPPROTO_UDP = C.IPPROTO_UDP + _C_SOCK_DGRAM = C.SOCK_DGRAM + _C_SOCK_STREAM = C.SOCK_STREAM ) type ( diff --git a/src/net/cgo_unix_syscall.go b/src/net/cgo_unix_syscall.go index 735dcdfe..9cfc5783 100644 --- a/src/net/cgo_unix_syscall.go +++ b/src/net/cgo_unix_syscall.go @@ -14,19 +14,20 @@ import ( ) const ( - _C_AF_INET = syscall.AF_INET - _C_AF_INET6 = syscall.AF_INET6 - _C_AF_UNSPEC = syscall.AF_UNSPEC - _C_EAI_AGAIN = unix.EAI_AGAIN - _C_EAI_NONAME = unix.EAI_NONAME - _C_EAI_SERVICE = unix.EAI_SERVICE - _C_EAI_NODATA = unix.EAI_NODATA - _C_EAI_OVERFLOW = unix.EAI_OVERFLOW - _C_EAI_SYSTEM = unix.EAI_SYSTEM - _C_IPPROTO_TCP = syscall.IPPROTO_TCP - _C_IPPROTO_UDP = syscall.IPPROTO_UDP - _C_SOCK_DGRAM = syscall.SOCK_DGRAM - _C_SOCK_STREAM = syscall.SOCK_STREAM + _C_AF_INET = syscall.AF_INET + _C_AF_INET6 = syscall.AF_INET6 + _C_AF_UNSPEC = syscall.AF_UNSPEC + _C_EAI_ADDRFAMILY = unix.EAI_ADDRFAMILY + _C_EAI_AGAIN = unix.EAI_AGAIN + _C_EAI_NONAME = unix.EAI_NONAME + _C_EAI_SERVICE = unix.EAI_SERVICE + _C_EAI_NODATA = unix.EAI_NODATA + _C_EAI_OVERFLOW = unix.EAI_OVERFLOW + _C_EAI_SYSTEM = unix.EAI_SYSTEM + _C_IPPROTO_TCP = syscall.IPPROTO_TCP + _C_IPPROTO_UDP = syscall.IPPROTO_UDP + _C_SOCK_DGRAM = syscall.SOCK_DGRAM + _C_SOCK_STREAM = syscall.SOCK_STREAM ) type ( diff --git a/src/net/conf.go b/src/net/conf.go index 358f5434..92c5618d 100644 --- a/src/net/conf.go +++ b/src/net/conf.go @@ -13,7 +13,6 @@ import ( "os" "runtime" "sync" - "syscall" ) // The net package's name resolution is rather complicated. @@ -94,19 +93,30 @@ func initConfVal() { if confVal.dnsDebugLevel > 1 { println("go package net: confVal.netCgo =", confVal.netCgo, " netGo =", confVal.netGo) } + if dnsMode != "go" && dnsMode != "cgo" && dnsMode != "" { + println("go package net: GODEBUG=netdns contains an invalid dns mode, ignoring it") + } switch { - case confVal.netGo: - if netGoBuildTag { - println("go package net: built with netgo build tag; using Go's DNS resolver") + case netGoBuildTag || !cgoAvailable: + if dnsMode == "cgo" { + println("go package net: ignoring GODEBUG=netdns=cgo as the binary was compiled without support for the cgo resolver") } else { - println("go package net: GODEBUG setting forcing use of Go's resolver") + println("go package net: using the Go DNS resolver") + } + case netCgoBuildTag: + if dnsMode == "go" { + println("go package net: GODEBUG setting forcing use of the Go resolver") + } else { + println("go package net: using the cgo DNS resolver") } - case !cgoAvailable: - println("go package net: cgo resolver not supported; using Go's DNS resolver") - case confVal.netCgo || confVal.preferCgo: - println("go package net: using cgo DNS resolver") default: - println("go package net: dynamic selection of DNS resolver") + if dnsMode == "go" { + println("go package net: GODEBUG setting forcing use of the Go resolver") + } else if dnsMode == "cgo" { + println("go package net: GODEBUG setting forcing use of the cgo resolver") + } else { + println("go package net: dynamic selection of DNS resolver") + } } }() } @@ -138,7 +148,7 @@ func initConfVal() { // prefer the cgo resolver. // Note that LOCALDOMAIN can change behavior merely by being // specified with the empty string. - _, localDomainDefined := syscall.Getenv("LOCALDOMAIN") + _, localDomainDefined := os.LookupEnv("LOCALDOMAIN") if localDomainDefined || os.Getenv("RES_OPTIONS") != "" || os.Getenv("HOSTALIASES") != "" { confVal.preferCgo = true return diff --git a/src/net/dial.go b/src/net/dial.go index 28f346a3..db0404c3 100644 --- a/src/net/dial.go +++ b/src/net/dial.go @@ -25,46 +25,93 @@ const ( // defaultTCPKeepAliveCount is a default constant value for TCP_KEEPCNT. defaultTCPKeepAliveCount = 9 - // For the moment, MultiPath TCP is not used by default + // For the moment, MultiPath TCP is used by default with listeners, if + // available, but not with dialers. // See go.dev/issue/56539 - defaultMPTCPEnabled = false + defaultMPTCPEnabledListen = true + defaultMPTCPEnabledDial = false ) +// The type of service offered +// +// 0 == MPTCP disabled +// 1 == MPTCP enabled +// 2 == MPTCP enabled on listeners only +// 3 == MPTCP enabled on dialers only var multipathtcp = godebug.New("multipathtcp") -// mptcpStatus is a tristate for Multipath TCP, see go.dev/issue/56539 -type mptcpStatus uint8 +// mptcpStatusDial is a tristate for Multipath TCP on clients, +// see go.dev/issue/56539 +type mptcpStatusDial uint8 const ( - // The value 0 is the system default, linked to defaultMPTCPEnabled - mptcpUseDefault mptcpStatus = iota - mptcpEnabled - mptcpDisabled + // The value 0 is the system default, linked to defaultMPTCPEnabledDial + mptcpUseDefaultDial mptcpStatusDial = iota + mptcpEnabledDial + mptcpDisabledDial ) -func (m *mptcpStatus) get() bool { +func (m *mptcpStatusDial) get() bool { switch *m { - case mptcpEnabled: + case mptcpEnabledDial: return true - case mptcpDisabled: + case mptcpDisabledDial: return false } // If MPTCP is forced via GODEBUG=multipathtcp=1 - if multipathtcp.Value() == "1" { + if multipathtcp.Value() == "1" || multipathtcp.Value() == "3" { multipathtcp.IncNonDefault() return true } - return defaultMPTCPEnabled + return defaultMPTCPEnabledDial } -func (m *mptcpStatus) set(use bool) { +func (m *mptcpStatusDial) set(use bool) { if use { - *m = mptcpEnabled + *m = mptcpEnabledDial } else { - *m = mptcpDisabled + *m = mptcpDisabledDial + } +} + +// mptcpStatusListen is a tristate for Multipath TCP on servers, +// see go.dev/issue/56539 +type mptcpStatusListen uint8 + +const ( + // The value 0 is the system default, linked to defaultMPTCPEnabledListen + mptcpUseDefaultListen mptcpStatusListen = iota + mptcpEnabledListen + mptcpDisabledListen +) + +func (m *mptcpStatusListen) get() bool { + switch *m { + case mptcpEnabledListen: + return true + case mptcpDisabledListen: + return false + } + + // If MPTCP is disabled via GODEBUG=multipathtcp=0 or only + // enabled on dialers, but not on listeners. + if multipathtcp.Value() == "0" || multipathtcp.Value() == "3" { + multipathtcp.IncNonDefault() + + return false + } + + return defaultMPTCPEnabledListen +} + +func (m *mptcpStatusListen) set(use bool) { + if use { + *m = mptcpEnabledListen + } else { + *m = mptcpDisabledListen } } @@ -156,8 +203,10 @@ type Dialer struct { // connection but before actually dialing. // // Network and address parameters passed to Control function are not - // necessarily the ones passed to Dial. For example, passing "tcp" to Dial - // will cause the Control function to be called with "tcp4" or "tcp6". + // necessarily the ones passed to Dial. Calling Dial with TCP networks + // will cause the Control function to be called with "tcp4" or "tcp6", + // UDP networks become "udp4" or "udp6", IP networks become "ip4" or "ip6", + // and other known networks are passed as-is. // // Control is ignored if ControlContext is not nil. Control func(network, address string, c syscall.RawConn) error @@ -166,8 +215,10 @@ type Dialer struct { // connection but before actually dialing. // // Network and address parameters passed to ControlContext function are not - // necessarily the ones passed to Dial. For example, passing "tcp" to Dial - // will cause the ControlContext function to be called with "tcp4" or "tcp6". + // necessarily the ones passed to Dial. Calling Dial with TCP networks + // will cause the ControlContext function to be called with "tcp4" or "tcp6", + // UDP networks become "udp4" or "udp6", IP networks become "ip4" or "ip6", + // and other known networks are passed as-is. // // If ControlContext is not nil, Control is ignored. ControlContext func(ctx context.Context, network, address string, c syscall.RawConn) error @@ -175,7 +226,7 @@ type Dialer struct { // If mptcpStatus is set to a value allowing Multipath TCP (MPTCP) to be // used, any call to Dial with "tcp(4|6)" as network will use MPTCP if // supported by the operating system. - mptcpStatus mptcpStatus + mptcpStatus mptcpStatusDial } func (d *Dialer) dualStack() bool { return d.FallbackDelay >= 0 } @@ -692,9 +743,11 @@ type ListenConfig struct { // If Control is not nil, it is called after creating the network // connection but before binding it to the operating system. // - // Network and address parameters passed to Control method are not - // necessarily the ones passed to Listen. For example, passing "tcp" to - // Listen will cause the Control function to be called with "tcp4" or "tcp6". + // Network and address parameters passed to Control function are not + // necessarily the ones passed to Listen. Calling Listen with TCP networks + // will cause the Control function to be called with "tcp4" or "tcp6", + // UDP networks become "udp4" or "udp6", IP networks become "ip4" or "ip6", + // and other known networks are passed as-is. Control func(network, address string, c syscall.RawConn) error // KeepAlive specifies the keep-alive period for network @@ -720,7 +773,7 @@ type ListenConfig struct { // If mptcpStatus is set to a value allowing Multipath TCP (MPTCP) to be // used, any call to Listen with "tcp(4|6)" as network will use MPTCP if // supported by the operating system. - mptcpStatus mptcpStatus + mptcpStatus mptcpStatusListen } // MultipathTCP reports whether MPTCP will be used. @@ -745,6 +798,9 @@ func (lc *ListenConfig) SetMultipathTCP(use bool) { // // See func Listen for a description of the network and address // parameters. +// +// The ctx argument is used while resolving the address on which to listen; +// it does not affect the returned Listener. func (lc *ListenConfig) Listen(ctx context.Context, network, address string) (Listener, error) { addrs, err := DefaultResolver.resolveAddrList(ctx, "listen", network, address, nil) if err != nil { @@ -779,6 +835,9 @@ func (lc *ListenConfig) Listen(ctx context.Context, network, address string) (Li // // See func ListenPacket for a description of the network and address // parameters. +// +// The ctx argument is used while resolving the address on which to listen; +// it does not affect the returned Listener. func (lc *ListenConfig) ListenPacket(ctx context.Context, network, address string) (PacketConn, error) { addrs, err := DefaultResolver.resolveAddrList(ctx, "listen", network, address, nil) if err != nil { diff --git a/src/net/dnsclient_unix.go b/src/net/dnsclient_unix.go index 54c7dc83..5e060a6b 100644 --- a/src/net/dnsclient_unix.go +++ b/src/net/dnsclient_unix.go @@ -18,6 +18,7 @@ import ( "internal/bytealg" "internal/godebug" "internal/itoa" + "internal/stringslite" "io" "os" "runtime" @@ -487,9 +488,7 @@ func avoidDNS(name string) bool { if name == "" { return true } - if name[len(name)-1] == '.' { - name = name[:len(name)-1] - } + name = stringslite.TrimSuffix(name, ".") return stringsHasSuffixFold(name, ".onion") } diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go index 31677573..c4e5194a 100644 --- a/src/net/dnsclient_unix_test.go +++ b/src/net/dnsclient_unix_test.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "maps" "os" "path" "path/filepath" @@ -429,7 +430,7 @@ func TestUpdateResolvConf(t *testing.T) { wg.Wait() } servers := conf.servers() - if !reflect.DeepEqual(servers, tt.servers) { + if !slices.Equal(servers, tt.servers) { t.Errorf("#%d: got %v; want %v", i, servers, tt.servers) continue } @@ -1154,7 +1155,7 @@ func testRotate(t *testing.T, rotate bool, nameservers, wantServers []string) { } } - if !reflect.DeepEqual(usedServers, wantServers) { + if !slices.Equal(usedServers, wantServers) { t.Errorf("rotate=%t got used servers:\n%v\nwant:\n%v", rotate, usedServers, wantServers) } } @@ -1433,7 +1434,7 @@ func TestStrictErrorsLookupIP(t *testing.T) { wantIPs[ip] = struct{}{} } } - if !reflect.DeepEqual(gotIPs, wantIPs) { + if !maps.Equal(gotIPs, wantIPs) { t.Errorf("#%d (%s) strict=%v: got ips %v; want %v", i, tt.desc, strict, gotIPs, wantIPs) } } @@ -1940,7 +1941,7 @@ func TestPTRandNonPTR(t *testing.T) { if err != nil { t.Fatalf("LookupAddr: %v", err) } - if want := []string{"golang.org."}; !reflect.DeepEqual(names, want) { + if want := []string{"golang.org."}; !slices.Equal(names, want) { t.Errorf("names = %q; want %q", names, want) } } @@ -2207,14 +2208,14 @@ func TestCVE202133195(t *testing.T) { if err.Error() != expectedErr.Error() { t.Fatalf("unexpected error: %s", err) } - if !reflect.DeepEqual(records, expected) { + if !slices.Equal(records, expected) { t.Error("Unexpected record set") } records, err = LookupAddr("192.0.2.42") if err.Error() != expectedErr.Error() { t.Fatalf("unexpected error: %s", err) } - if !reflect.DeepEqual(records, expected) { + if !slices.Equal(records, expected) { t.Error("Unexpected record set") } }, diff --git a/src/net/dnsconfig_unix_test.go b/src/net/dnsconfig_unix_test.go index 0aae2ba8..4db1c5a4 100644 --- a/src/net/dnsconfig_unix_test.go +++ b/src/net/dnsconfig_unix_test.go @@ -11,6 +11,7 @@ import ( "io/fs" "os" "reflect" + "slices" "strings" "testing" "time" @@ -250,7 +251,7 @@ func TestDNSDefaultSearch(t *testing.T) { for _, tt := range dnsDefaultSearchTests { getHostname = func() (string, error) { return tt.name, tt.err } got := dnsDefaultSearch() - if !reflect.DeepEqual(got, tt.want) { + if !slices.Equal(got, tt.want) { t.Errorf("dnsDefaultSearch with hostname %q and error %+v = %q, wanted %q", tt.name, tt.err, got, tt.want) } } diff --git a/src/net/example_test.go b/src/net/example_test.go index 2c045d73..12c83970 100644 --- a/src/net/example_test.go +++ b/src/net/example_test.go @@ -334,7 +334,7 @@ func ExampleIP_To16() { // 10.255.0.0 } -func ExampleIP_to4() { +func ExampleIP_To4() { ipv6 := net.IP{0xfc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} ipv4 := net.IPv4(10, 255, 0, 0) diff --git a/src/net/hosts_test.go b/src/net/hosts_test.go index 5f229207..2661e79d 100644 --- a/src/net/hosts_test.go +++ b/src/net/hosts_test.go @@ -5,7 +5,7 @@ package net import ( - "reflect" + "slices" "strings" "testing" ) @@ -73,7 +73,7 @@ func testStaticHost(t *testing.T, hostsPath string, ent staticHostEntry) { ins := []string{ent.in, absDomainName(ent.in), strings.ToLower(ent.in), strings.ToUpper(ent.in)} for _, in := range ins { addrs, _ := lookupStaticHost(in) - if !reflect.DeepEqual(addrs, ent.out) { + if !slices.Equal(addrs, ent.out) { t.Errorf("%s, lookupStaticHost(%s) = %v; want %v", hostsPath, in, addrs, ent.out) } } @@ -143,7 +143,7 @@ func testStaticAddr(t *testing.T, hostsPath string, ent staticHostEntry) { for i := range ent.out { ent.out[i] = absDomainName(ent.out[i]) } - if !reflect.DeepEqual(hosts, ent.out) { + if !slices.Equal(hosts, ent.out) { t.Errorf("%s, lookupStaticAddr(%s) = %v; want %v", hostsPath, ent.in, hosts, ent.out) } } diff --git a/src/net/http/async_test.go b/src/net/http/async_test.go new file mode 100644 index 00000000..545cbcf5 --- /dev/null +++ b/src/net/http/async_test.go @@ -0,0 +1,52 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "errors" + "internal/synctest" +) + +var errStillRunning = errors.New("async op still running") + +type asyncResult[T any] struct { + donec chan struct{} + res T + err error +} + +// runAsync runs f in a new goroutine. +// It returns an asyncResult which acts as a future. +// +// Must be called from within a synctest bubble. +func runAsync[T any](f func() (T, error)) *asyncResult[T] { + r := &asyncResult[T]{ + donec: make(chan struct{}), + } + go func() { + defer close(r.donec) + r.res, r.err = f() + }() + synctest.Wait() + return r +} + +// done reports whether the function has returned. +func (r *asyncResult[T]) done() bool { + _, err := r.result() + return err != errStillRunning +} + +// result returns the result of the function. +// If the function hasn't completed yet, it returns errStillRunning. +func (r *asyncResult[T]) result() (T, error) { + select { + case <-r.donec: + return r.res, r.err + default: + var zero T + return zero, errStillRunning + } +} diff --git a/src/net/http/cgi/cgi_main.go b/src/net/http/cgi/cgi_main.go index 033036d0..1435f1b7 100644 --- a/src/net/http/cgi/cgi_main.go +++ b/src/net/http/cgi/cgi_main.go @@ -7,6 +7,7 @@ package cgi import ( "fmt" "io" + "maps" "net/http" "os" "path" @@ -63,22 +64,12 @@ func testCGI() { fmt.Printf("test=Hello CGI\r\n") - keys := make([]string, 0, len(params)) - for k := range params { - keys = append(keys, k) - } - slices.Sort(keys) - for _, key := range keys { + for _, key := range slices.Sorted(maps.Keys(params)) { fmt.Printf("param-%s=%s\r\n", key, params.Get(key)) } envs := envMap(os.Environ()) - keys = make([]string, 0, len(envs)) - for k := range envs { - keys = append(keys, k) - } - slices.Sort(keys) - for _, key := range keys { + for _, key := range slices.Sorted(maps.Keys(envs)) { fmt.Printf("env-%s=%s\r\n", key, envs[key]) } diff --git a/src/net/http/cgi/host_test.go b/src/net/http/cgi/host_test.go index 7fe0e625..8ecfa19f 100644 --- a/src/net/http/cgi/host_test.go +++ b/src/net/http/cgi/host_test.go @@ -16,9 +16,9 @@ import ( "net/http/httptest" "os" "path/filepath" - "reflect" "regexp" "runtime" + "slices" "strings" "testing" "time" @@ -510,7 +510,7 @@ func TestRemoveLeadingDuplicates(t *testing.T) { } for _, tt := range tests { got := removeLeadingDuplicates(tt.env) - if !reflect.DeepEqual(got, tt.want) { + if !slices.Equal(got, tt.want) { t.Errorf("removeLeadingDuplicates(%q) = %q; want %q", tt.env, got, tt.want) } } diff --git a/src/net/http/client.go b/src/net/http/client.go index f8892c2b..9231f63e 100644 --- a/src/net/http/client.go +++ b/src/net/http/client.go @@ -388,15 +388,12 @@ func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTi } stopTimerCh := make(chan struct{}) - var once sync.Once - stopTimer = func() { - once.Do(func() { - close(stopTimerCh) - if cancelCtx != nil { - cancelCtx() - } - }) - } + stopTimer = sync.OnceFunc(func() { + close(stopTimerCh) + if cancelCtx != nil { + cancelCtx() + } + }) timer := time.NewTimer(time.Until(deadline)) var timedOut atomic.Bool @@ -737,11 +734,16 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { return nil, uerr(err) } - var shouldRedirect bool - redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0]) + var shouldRedirect, includeBodyOnHop bool + redirectMethod, shouldRedirect, includeBodyOnHop = redirectBehavior(req.Method, resp, reqs[0]) if !shouldRedirect { return resp, nil } + if !includeBodyOnHop { + // Once a hop drops the body, we never send it again + // (because we're now handling a redirect for a request with no body). + includeBody = false + } req.closeBody() } diff --git a/src/net/http/client_test.go b/src/net/http/client_test.go index d57096fc..1ce95395 100644 --- a/src/net/http/client_test.go +++ b/src/net/http/client_test.go @@ -346,7 +346,7 @@ func TestPostRedirects(t *testing.T) { `POST /?code=307&next=303,308,302 "c307"`, `POST /?code=303&next=308,302 "c307"`, `GET /?code=308&next=302 ""`, - `GET /?code=302 "c307"`, + `GET /?code=302 ""`, `GET / ""`, `POST /?code=308&next=302,301 "c308"`, `POST /?code=302&next=301 "c308"`, @@ -376,7 +376,7 @@ func TestDeleteRedirects(t *testing.T) { `DELETE /?code=301&next=302,308 "c301"`, `GET /?code=302&next=308 ""`, `GET /?code=308 ""`, - `GET / "c301"`, + `GET / ""`, `DELETE /?code=302&next=302 "c302"`, `GET /?code=302 ""`, `GET / ""`, @@ -385,7 +385,7 @@ func TestDeleteRedirects(t *testing.T) { `DELETE /?code=307&next=301,308,303,302,304 "c307"`, `DELETE /?code=301&next=308,303,302,304 "c307"`, `GET /?code=308&next=303,302,304 ""`, - `GET /?code=303&next=302,304 "c307"`, + `GET /?code=303&next=302,304 ""`, `GET /?code=302&next=304 ""`, `GET /?code=304 ""`, `DELETE /?code=308&next=307 "c308"`, @@ -751,7 +751,7 @@ func testStreamingGet(t *testing.T, mode testMode) { var buf [10]byte for _, str := range []string{"i", "am", "also", "known", "as", "comet"} { say <- str - n, err := io.ReadFull(res.Body, buf[0:len(str)]) + n, err := io.ReadFull(res.Body, buf[:len(str)]) if err != nil { t.Fatalf("ReadFull on %q: %v", str, err) } diff --git a/src/net/http/clientserver_test.go b/src/net/http/clientserver_test.go index 3dc440dd..32d97ea9 100644 --- a/src/net/http/clientserver_test.go +++ b/src/net/http/clientserver_test.go @@ -15,8 +15,10 @@ import ( "crypto/tls" "fmt" "hash" + "internal/synctest" "io" "log" + "maps" "net" . "net/http" "net/http/httptest" @@ -38,9 +40,10 @@ import ( type testMode string const ( - http1Mode = testMode("h1") // HTTP/1.1 - https1Mode = testMode("https1") // HTTPS/1.1 - http2Mode = testMode("h2") // HTTP/2 + http1Mode = testMode("h1") // HTTP/1.1 + https1Mode = testMode("https1") // HTTPS/1.1 + http2Mode = testMode("h2") // HTTP/2 + http2UnencryptedMode = testMode("h2unencrypted") // HTTP/2 ) type testNotParallelOpt struct{} @@ -92,6 +95,37 @@ func run[T TBRun[T]](t T, f func(t T, mode testMode), opts ...any) { } } +// cleanupT wraps a testing.T and adds its own Cleanup method. +// Used to execute cleanup functions within a synctest bubble. +type cleanupT struct { + *testing.T + cleanups []func() +} + +// Cleanup replaces T.Cleanup. +func (t *cleanupT) Cleanup(f func()) { + t.cleanups = append(t.cleanups, f) +} + +func (t *cleanupT) done() { + for _, f := range slices.Backward(t.cleanups) { + f() + } +} + +// runSynctest is run combined with synctest.Run. +// +// The TB passed to f arranges for cleanup functions to be run in the synctest bubble. +func runSynctest(t *testing.T, f func(t testing.TB, mode testMode), opts ...any) { + run(t, func(t *testing.T, mode testMode) { + synctest.Run(func() { + ct := &cleanupT{T: t} + defer ct.done() + f(ct, mode) + }) + }, opts...) +} + type clientServerTest struct { t testing.TB h2 bool @@ -99,6 +133,7 @@ type clientServerTest struct { ts *httptest.Server tr *Transport c *Client + li *fakeNetListener } func (t *clientServerTest) close() { @@ -136,6 +171,8 @@ func optWithServerLog(lg *log.Logger) func(*httptest.Server) { } } +var optFakeNet = new(struct{}) + // newClientServerTest creates and starts an httptest.Server. // // The mode parameter selects the implementation to test: @@ -147,6 +184,9 @@ func optWithServerLog(lg *log.Logger) func(*httptest.Server) { // // func(*httptest.Server) // run before starting the server // func(*http.Transport) +// +// The optFakeNet option configures the server and client to use a fake network implementation, +// suitable for use in testing/synctest tests. func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *clientServerTest { if mode == http2Mode { CondSkipHTTP2(t) @@ -156,9 +196,31 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c h2: mode == http2Mode, h: h, } - cst.ts = httptest.NewUnstartedServer(h) var transportFuncs []func(*Transport) + + if idx := slices.Index(opts, any(optFakeNet)); idx >= 0 { + opts = slices.Delete(opts, idx, idx+1) + cst.li = fakeNetListen() + cst.ts = &httptest.Server{ + Config: &Server{Handler: h}, + Listener: cst.li, + } + transportFuncs = append(transportFuncs, func(tr *Transport) { + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return cst.li.connect(), nil + } + }) + } else { + cst.ts = httptest.NewUnstartedServer(h) + } + + if mode == http2UnencryptedMode { + p := &Protocols{} + p.SetUnencryptedHTTP2(true) + cst.ts.Config.Protocols = p + } + for _, opt := range opts { switch opt := opt.(type) { case func(*Transport): @@ -179,6 +241,9 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c cst.ts.Start() case https1Mode: cst.ts.StartTLS() + case http2UnencryptedMode: + ExportHttp2ConfigureServer(cst.ts.Config, nil) + cst.ts.Start() case http2Mode: ExportHttp2ConfigureServer(cst.ts.Config, nil) cst.ts.TLS = cst.ts.Config.TLSConfig @@ -188,7 +253,7 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c } cst.c = cst.ts.Client() cst.tr = cst.c.Transport.(*Transport) - if mode == http2Mode { + if mode == http2Mode || mode == http2UnencryptedMode { if err := ExportHttp2ConfigureTransport(cst.tr); err != nil { t.Fatal(err) } @@ -196,6 +261,13 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c for _, f := range transportFuncs { f(cst.tr) } + + if mode == http2UnencryptedMode { + p := &Protocols{} + p.SetUnencryptedHTTP2(true) + cst.tr.Protocols = p + } + t.Cleanup(func() { cst.close() }) @@ -213,9 +285,19 @@ func (w testLogWriter) Write(b []byte) (int, error) { // Testing the newClientServerTest helper itself. func TestNewClientServerTest(t *testing.T) { - run(t, testNewClientServerTest, []testMode{http1Mode, https1Mode, http2Mode}) + modes := []testMode{http1Mode, https1Mode, http2Mode} + t.Run("realnet", func(t *testing.T) { + run(t, func(t *testing.T, mode testMode) { + testNewClientServerTest(t, mode) + }, modes) + }) + t.Run("synctest", func(t *testing.T) { + runSynctest(t, func(t testing.TB, mode testMode) { + testNewClientServerTest(t, mode, optFakeNet) + }, modes) + }) } -func testNewClientServerTest(t *testing.T, mode testMode) { +func testNewClientServerTest(t testing.TB, mode testMode, opts ...any) { var got struct { sync.Mutex proto string @@ -227,7 +309,7 @@ func testNewClientServerTest(t *testing.T, mode testMode) { got.proto = r.Proto got.hasTLS = r.TLS != nil }) - cst := newClientServerTest(t, mode, h) + cst := newClientServerTest(t, mode, h, opts...) if _, err := cst.c.Head(cst.ts.URL); err != nil { t.Fatal(err) } @@ -274,7 +356,7 @@ func testChunkedResponseHeaders(t *testing.T, mode testMode) { if mode == http2Mode { wantTE = nil } - if !reflect.DeepEqual(res.TransferEncoding, wantTE) { + if !slices.Equal(res.TransferEncoding, wantTE) { t.Errorf("TransferEncoding = %v; want %v", res.TransferEncoding, wantTE) } if got, haveCL := res.Header["Content-Length"]; haveCL { @@ -689,12 +771,6 @@ func testCancelRequestMidBody(t *testing.T, mode testMode) { func TestTrailersClientToServer(t *testing.T) { run(t, testTrailersClientToServer) } func testTrailersClientToServer(t *testing.T, mode testMode) { cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - var decl []string - for k := range r.Trailer { - decl = append(decl, k) - } - slices.Sort(decl) - slurp, err := io.ReadAll(r.Body) if err != nil { t.Errorf("Server reading request body: %v", err) @@ -705,6 +781,7 @@ func testTrailersClientToServer(t *testing.T, mode testMode) { if r.Trailer == nil { io.WriteString(w, "nil Trailer") } else { + decl := slices.Sorted(maps.Keys(r.Trailer)) fmt.Fprintf(w, "decl: %v, vals: %s, %s", decl, r.Trailer.Get("Client-Trailer-A"), @@ -1602,6 +1679,7 @@ func testBidiStreamReverseProxy(t *testing.T, mode testMode) { _, err := io.CopyN(io.MultiWriter(h, pw), rand.Reader, size) go pw.Close() if err != nil { + t.Errorf("body copy: %v", err) bodyRes <- err } else { bodyRes <- h diff --git a/src/net/http/cookiejar/jar.go b/src/net/http/cookiejar/jar.go index 2eec1a3e..edf14d03 100644 --- a/src/net/http/cookiejar/jar.go +++ b/src/net/http/cookiejar/jar.go @@ -500,9 +500,7 @@ func (j *Jar) domainAndType(host, domain string) (string, bool, error) { // From here on: If the cookie is valid, it is a domain cookie (with // the one exception of a public suffix below). // See RFC 6265 section 5.2.3. - if domain[0] == '.' { - domain = domain[1:] - } + domain = strings.TrimPrefix(domain, ".") if len(domain) == 0 || domain[0] == '.' { // Received either "Domain=." or "Domain=..some.thing", diff --git a/src/net/http/example_filesystem_test.go b/src/net/http/example_filesystem_test.go index 0e81458a..ebcb994f 100644 --- a/src/net/http/example_filesystem_test.go +++ b/src/net/http/example_filesystem_test.go @@ -5,6 +5,7 @@ package http_test import ( + "io" "io/fs" "log" "net/http" @@ -40,6 +41,9 @@ func (f dotFileHidingFile) Readdir(n int) (fis []fs.FileInfo, err error) { fis = append(fis, file) } } + if err == nil && n > 0 && len(fis) == 0 { + err = io.EOF + } return } diff --git a/src/net/http/example_test.go b/src/net/http/example_test.go index 2f411d1d..f40273f1 100644 --- a/src/net/http/example_test.go +++ b/src/net/http/example_test.go @@ -193,3 +193,31 @@ func ExampleNotFoundHandler() { log.Fatal(http.ListenAndServe(":8080", mux)) } + +func ExampleProtocols_http1() { + srv := http.Server{ + Addr: ":8443", + } + + // Serve only HTTP/1. + srv.Protocols = new(http.Protocols) + srv.Protocols.SetHTTP1(true) + + log.Fatal(srv.ListenAndServeTLS("cert.pem", "key.pem")) +} + +func ExampleProtocols_http1or2() { + t := http.DefaultTransport.(*http.Transport).Clone() + + // Use either HTTP/1 and HTTP/2. + t.Protocols = new(http.Protocols) + t.Protocols.SetHTTP1(true) + t.Protocols.SetHTTP2(true) + + cli := &http.Client{Transport: t} + res, err := cli.Get("http://www.google.com/robots.txt") + if err != nil { + log.Fatal(err) + } + res.Body.Close() +} diff --git a/src/net/http/fs_test.go b/src/net/http/fs_test.go index 2ffffbf0..3149ca35 100644 --- a/src/net/http/fs_test.go +++ b/src/net/http/fs_test.go @@ -24,9 +24,9 @@ import ( "os/exec" "path" "path/filepath" - "reflect" "regexp" "runtime" + "slices" "strconv" "strings" "testing" @@ -516,7 +516,7 @@ func testServeFileContentType(t *testing.T, mode testMode) { if err != nil { t.Fatal(err) } - if h := resp.Header["Content-Type"]; !reflect.DeepEqual(h, want) { + if h := resp.Header["Content-Type"]; !slices.Equal(h, want) { t.Errorf("Content-Type mismatch: got %v, want %v", h, want) } resp.Body.Close() @@ -1448,7 +1448,7 @@ func TestFileServerCleanPath(t *testing.T) { rr := httptest.NewRecorder() req, _ := NewRequest("GET", "http://foo.localhost"+tt.path, nil) FileServer(fileServerCleanPathDir{&log}).ServeHTTP(rr, req) - if !reflect.DeepEqual(log, tt.wantOpen) { + if !slices.Equal(log, tt.wantOpen) { t.Logf("For %s: Opens = %q; want %q", tt.path, log, tt.wantOpen) } if rr.Code != tt.wantCode { diff --git a/src/net/http/h2_bundle.go b/src/net/http/h2_bundle.go index 0b305844..22f013f1 100644 --- a/src/net/http/h2_bundle.go +++ b/src/net/http/h2_bundle.go @@ -883,7 +883,7 @@ func (c *http2dialCall) dial(ctx context.Context, addr string) { // This code decides which ones live or die. // The return value used is whether c was used. // c is never closed. -func (p *http2clientConnPool) addConnIfNeeded(key string, t *http2Transport, c *tls.Conn) (used bool, err error) { +func (p *http2clientConnPool) addConnIfNeeded(key string, t *http2Transport, c net.Conn) (used bool, err error) { p.mu.Lock() for _, cc := range p.conns[key] { if cc.CanTakeNewRequest() { @@ -919,8 +919,8 @@ type http2addConnCall struct { err error } -func (c *http2addConnCall) run(t *http2Transport, key string, tc *tls.Conn) { - cc, err := t.NewClientConn(tc) +func (c *http2addConnCall) run(t *http2Transport, key string, nc net.Conn) { + cc, err := t.NewClientConn(nc) p := c.p p.mu.Lock() @@ -1035,6 +1035,169 @@ func http2shouldRetryDial(call *http2dialCall, req *Request) bool { return call.ctx.Err() != nil } +// http2Config is a package-internal version of net/http.HTTP2Config. +// +// http.HTTP2Config was added in Go 1.24. +// When running with a version of net/http that includes HTTP2Config, +// we merge the configuration with the fields in Transport or Server +// to produce an http2Config. +// +// Zero valued fields in http2Config are interpreted as in the +// net/http.HTTPConfig documentation. +// +// Precedence order for reconciling configurations is: +// +// - Use the net/http.{Server,Transport}.HTTP2Config value, when non-zero. +// - Otherwise use the http2.{Server.Transport} value. +// - If the resulting value is zero or out of range, use a default. +type http2http2Config struct { + MaxConcurrentStreams uint32 + MaxDecoderHeaderTableSize uint32 + MaxEncoderHeaderTableSize uint32 + MaxReadFrameSize uint32 + MaxUploadBufferPerConnection int32 + MaxUploadBufferPerStream int32 + SendPingTimeout time.Duration + PingTimeout time.Duration + WriteByteTimeout time.Duration + PermitProhibitedCipherSuites bool + CountError func(errType string) +} + +// configFromServer merges configuration settings from +// net/http.Server.HTTP2Config and http2.Server. +func http2configFromServer(h1 *Server, h2 *http2Server) http2http2Config { + conf := http2http2Config{ + MaxConcurrentStreams: h2.MaxConcurrentStreams, + MaxEncoderHeaderTableSize: h2.MaxEncoderHeaderTableSize, + MaxDecoderHeaderTableSize: h2.MaxDecoderHeaderTableSize, + MaxReadFrameSize: h2.MaxReadFrameSize, + MaxUploadBufferPerConnection: h2.MaxUploadBufferPerConnection, + MaxUploadBufferPerStream: h2.MaxUploadBufferPerStream, + SendPingTimeout: h2.ReadIdleTimeout, + PingTimeout: h2.PingTimeout, + WriteByteTimeout: h2.WriteByteTimeout, + PermitProhibitedCipherSuites: h2.PermitProhibitedCipherSuites, + CountError: h2.CountError, + } + http2fillNetHTTPServerConfig(&conf, h1) + http2setConfigDefaults(&conf, true) + return conf +} + +// configFromServer merges configuration settings from h2 and h2.t1.HTTP2 +// (the net/http Transport). +func http2configFromTransport(h2 *http2Transport) http2http2Config { + conf := http2http2Config{ + MaxEncoderHeaderTableSize: h2.MaxEncoderHeaderTableSize, + MaxDecoderHeaderTableSize: h2.MaxDecoderHeaderTableSize, + MaxReadFrameSize: h2.MaxReadFrameSize, + SendPingTimeout: h2.ReadIdleTimeout, + PingTimeout: h2.PingTimeout, + WriteByteTimeout: h2.WriteByteTimeout, + } + + // Unlike most config fields, where out-of-range values revert to the default, + // Transport.MaxReadFrameSize clips. + if conf.MaxReadFrameSize < http2minMaxFrameSize { + conf.MaxReadFrameSize = http2minMaxFrameSize + } else if conf.MaxReadFrameSize > http2maxFrameSize { + conf.MaxReadFrameSize = http2maxFrameSize + } + + if h2.t1 != nil { + http2fillNetHTTPTransportConfig(&conf, h2.t1) + } + http2setConfigDefaults(&conf, false) + return conf +} + +func http2setDefault[T ~int | ~int32 | ~uint32 | ~int64](v *T, minval, maxval, defval T) { + if *v < minval || *v > maxval { + *v = defval + } +} + +func http2setConfigDefaults(conf *http2http2Config, server bool) { + http2setDefault(&conf.MaxConcurrentStreams, 1, math.MaxUint32, http2defaultMaxStreams) + http2setDefault(&conf.MaxEncoderHeaderTableSize, 1, math.MaxUint32, http2initialHeaderTableSize) + http2setDefault(&conf.MaxDecoderHeaderTableSize, 1, math.MaxUint32, http2initialHeaderTableSize) + if server { + http2setDefault(&conf.MaxUploadBufferPerConnection, http2initialWindowSize, math.MaxInt32, 1<<20) + } else { + http2setDefault(&conf.MaxUploadBufferPerConnection, http2initialWindowSize, math.MaxInt32, http2transportDefaultConnFlow) + } + if server { + http2setDefault(&conf.MaxUploadBufferPerStream, 1, math.MaxInt32, 1<<20) + } else { + http2setDefault(&conf.MaxUploadBufferPerStream, 1, math.MaxInt32, http2transportDefaultStreamFlow) + } + http2setDefault(&conf.MaxReadFrameSize, http2minMaxFrameSize, http2maxFrameSize, http2defaultMaxReadFrameSize) + http2setDefault(&conf.PingTimeout, 1, math.MaxInt64, 15*time.Second) +} + +// adjustHTTP1MaxHeaderSize converts a limit in bytes on the size of an HTTP/1 header +// to an HTTP/2 MAX_HEADER_LIST_SIZE value. +func http2adjustHTTP1MaxHeaderSize(n int64) int64 { + // http2's count is in a slightly different unit and includes 32 bytes per pair. + // So, take the net/http.Server value and pad it up a bit, assuming 10 headers. + const perFieldOverhead = 32 // per http2 spec + const typicalHeaders = 10 // conservative + return n + typicalHeaders*perFieldOverhead +} + +// fillNetHTTPServerConfig sets fields in conf from srv.HTTP2. +func http2fillNetHTTPServerConfig(conf *http2http2Config, srv *Server) { + http2fillNetHTTPConfig(conf, srv.HTTP2) +} + +// fillNetHTTPServerConfig sets fields in conf from tr.HTTP2. +func http2fillNetHTTPTransportConfig(conf *http2http2Config, tr *Transport) { + http2fillNetHTTPConfig(conf, tr.HTTP2) +} + +func http2fillNetHTTPConfig(conf *http2http2Config, h2 *HTTP2Config) { + if h2 == nil { + return + } + if h2.MaxConcurrentStreams != 0 { + conf.MaxConcurrentStreams = uint32(h2.MaxConcurrentStreams) + } + if h2.MaxEncoderHeaderTableSize != 0 { + conf.MaxEncoderHeaderTableSize = uint32(h2.MaxEncoderHeaderTableSize) + } + if h2.MaxDecoderHeaderTableSize != 0 { + conf.MaxDecoderHeaderTableSize = uint32(h2.MaxDecoderHeaderTableSize) + } + if h2.MaxConcurrentStreams != 0 { + conf.MaxConcurrentStreams = uint32(h2.MaxConcurrentStreams) + } + if h2.MaxReadFrameSize != 0 { + conf.MaxReadFrameSize = uint32(h2.MaxReadFrameSize) + } + if h2.MaxReceiveBufferPerConnection != 0 { + conf.MaxUploadBufferPerConnection = int32(h2.MaxReceiveBufferPerConnection) + } + if h2.MaxReceiveBufferPerStream != 0 { + conf.MaxUploadBufferPerStream = int32(h2.MaxReceiveBufferPerStream) + } + if h2.SendPingTimeout != 0 { + conf.SendPingTimeout = h2.SendPingTimeout + } + if h2.PingTimeout != 0 { + conf.PingTimeout = h2.PingTimeout + } + if h2.WriteByteTimeout != 0 { + conf.WriteByteTimeout = h2.WriteByteTimeout + } + if h2.PermitProhibitedCipherSuites { + conf.PermitProhibitedCipherSuites = true + } + if h2.CountError != nil { + conf.CountError = h2.CountError + } +} + // Buffer chunks are allocated from a pool to reduce pressure on GC. // The maximum wasted space per dataBuffer is 2x the largest size class, // which happens when the dataBuffer has multiple chunks and there is @@ -2898,7 +3061,7 @@ func (mh *http2MetaHeadersFrame) checkPseudos() error { pf := mh.PseudoFields() for i, hf := range pf { switch hf.Name { - case ":method", ":path", ":scheme", ":authority": + case ":method", ":path", ":scheme", ":authority", ":protocol": isRequest = true case ":status": isResponse = true @@ -2906,7 +3069,7 @@ func (mh *http2MetaHeadersFrame) checkPseudos() error { return http2pseudoHeaderError(hf.Name) } // Check for duplicates. - // This would be a bad algorithm, but N is 4. + // This would be a bad algorithm, but N is 5. // And this doesn't allocate. for _, hf2 := range pf[:i] { if hf.Name == hf2.Name { @@ -3350,6 +3513,15 @@ var ( http2logFrameWrites bool http2logFrameReads bool http2inTests bool + + // Enabling extended CONNECT by causes browsers to attempt to use + // WebSockets-over-HTTP/2. This results in problems when the server's websocket + // package doesn't support extended CONNECT. + // + // Disable extended CONNECT by default for now. + // + // Issue #71128. + http2disableExtendedConnectProtocol = true ) func init() { @@ -3362,6 +3534,9 @@ func init() { http2logFrameWrites = true http2logFrameReads = true } + if strings.Contains(e, "http2xconnect=1") { + http2disableExtendedConnectProtocol = false + } } const ( @@ -3453,6 +3628,10 @@ func (s http2Setting) Valid() error { if s.Val < 16384 || s.Val > 1<<24-1 { return http2ConnectionError(http2ErrCodeProtocol) } + case http2SettingEnableConnectProtocol: + if s.Val != 1 && s.Val != 0 { + return http2ConnectionError(http2ErrCodeProtocol) + } } return nil } @@ -3462,21 +3641,23 @@ func (s http2Setting) Valid() error { type http2SettingID uint16 const ( - http2SettingHeaderTableSize http2SettingID = 0x1 - http2SettingEnablePush http2SettingID = 0x2 - http2SettingMaxConcurrentStreams http2SettingID = 0x3 - http2SettingInitialWindowSize http2SettingID = 0x4 - http2SettingMaxFrameSize http2SettingID = 0x5 - http2SettingMaxHeaderListSize http2SettingID = 0x6 + http2SettingHeaderTableSize http2SettingID = 0x1 + http2SettingEnablePush http2SettingID = 0x2 + http2SettingMaxConcurrentStreams http2SettingID = 0x3 + http2SettingInitialWindowSize http2SettingID = 0x4 + http2SettingMaxFrameSize http2SettingID = 0x5 + http2SettingMaxHeaderListSize http2SettingID = 0x6 + http2SettingEnableConnectProtocol http2SettingID = 0x8 ) var http2settingName = map[http2SettingID]string{ - http2SettingHeaderTableSize: "HEADER_TABLE_SIZE", - http2SettingEnablePush: "ENABLE_PUSH", - http2SettingMaxConcurrentStreams: "MAX_CONCURRENT_STREAMS", - http2SettingInitialWindowSize: "INITIAL_WINDOW_SIZE", - http2SettingMaxFrameSize: "MAX_FRAME_SIZE", - http2SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE", + http2SettingHeaderTableSize: "HEADER_TABLE_SIZE", + http2SettingEnablePush: "ENABLE_PUSH", + http2SettingMaxConcurrentStreams: "MAX_CONCURRENT_STREAMS", + http2SettingInitialWindowSize: "INITIAL_WINDOW_SIZE", + http2SettingMaxFrameSize: "MAX_FRAME_SIZE", + http2SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE", + http2SettingEnableConnectProtocol: "ENABLE_CONNECT_PROTOCOL", } func (s http2SettingID) String() string { @@ -3550,13 +3731,19 @@ func (cw http2closeWaiter) Wait() { // Its buffered writer is lazily allocated as needed, to minimize // idle memory usage with many connections. type http2bufferedWriter struct { - _ http2incomparable - w io.Writer // immutable - bw *bufio.Writer // non-nil when data is buffered + _ http2incomparable + group http2synctestGroupInterface // immutable + conn net.Conn // immutable + bw *bufio.Writer // non-nil when data is buffered + byteTimeout time.Duration // immutable, WriteByteTimeout } -func http2newBufferedWriter(w io.Writer) *http2bufferedWriter { - return &http2bufferedWriter{w: w} +func http2newBufferedWriter(group http2synctestGroupInterface, conn net.Conn, timeout time.Duration) *http2bufferedWriter { + return &http2bufferedWriter{ + group: group, + conn: conn, + byteTimeout: timeout, + } } // bufWriterPoolBufferSize is the size of bufio.Writer's @@ -3583,7 +3770,7 @@ func (w *http2bufferedWriter) Available() int { func (w *http2bufferedWriter) Write(p []byte) (n int, err error) { if w.bw == nil { bw := http2bufWriterPool.Get().(*bufio.Writer) - bw.Reset(w.w) + bw.Reset((*http2bufferedWriterTimeoutWriter)(w)) w.bw = bw } return w.bw.Write(p) @@ -3601,6 +3788,38 @@ func (w *http2bufferedWriter) Flush() error { return err } +type http2bufferedWriterTimeoutWriter http2bufferedWriter + +func (w *http2bufferedWriterTimeoutWriter) Write(p []byte) (n int, err error) { + return http2writeWithByteTimeout(w.group, w.conn, w.byteTimeout, p) +} + +// writeWithByteTimeout writes to conn. +// If more than timeout passes without any bytes being written to the connection, +// the write fails. +func http2writeWithByteTimeout(group http2synctestGroupInterface, conn net.Conn, timeout time.Duration, p []byte) (n int, err error) { + if timeout <= 0 { + return conn.Write(p) + } + for { + var now time.Time + if group == nil { + now = time.Now() + } else { + now = group.Now() + } + conn.SetWriteDeadline(now.Add(timeout)) + nn, err := conn.Write(p[n:]) + n += nn + if n == len(p) || nn == 0 || !errors.Is(err, os.ErrDeadlineExceeded) { + // Either we finished the write, made no progress, or hit the deadline. + // Whichever it is, we're done now. + conn.SetWriteDeadline(time.Time{}) + return n, err + } + } +} + func http2mustUint31(v int32) uint32 { if v < 0 || v > 2147483647 { panic("out of range") @@ -3882,10 +4101,14 @@ func (p *http2pipe) Done() <-chan struct{} { } const ( - http2prefaceTimeout = 10 * time.Second - http2firstSettingsTimeout = 2 * time.Second // should be in-flight with preface anyway - http2handlerChunkWriteSize = 4 << 10 - http2defaultMaxStreams = 250 // TODO: make this 100 as the GFE seems to? + http2prefaceTimeout = 10 * time.Second + http2firstSettingsTimeout = 2 * time.Second // should be in-flight with preface anyway + http2handlerChunkWriteSize = 4 << 10 + http2defaultMaxStreams = 250 // TODO: make this 100 as the GFE seems to? + + // maxQueuedControlFrames is the maximum number of control frames like + // SETTINGS, PING and RST_STREAM that will be queued for writing before + // the connection is closed to prevent memory exhaustion attacks. http2maxQueuedControlFrames = 10000 ) @@ -3957,6 +4180,22 @@ type http2Server struct { // If zero or negative, there is no timeout. IdleTimeout time.Duration + // ReadIdleTimeout is the timeout after which a health check using a ping + // frame will be carried out if no frame is received on the connection. + // If zero, no health check is performed. + ReadIdleTimeout time.Duration + + // PingTimeout is the timeout after which the connection will be closed + // if a response to a ping is not received. + // If zero, a default of 15 seconds is used. + PingTimeout time.Duration + + // WriteByteTimeout is the timeout after which a connection will be + // closed if no data can be written to it. The timeout begins when data is + // available to write, and is extended whenever any bytes are written. + // If zero or negative, there is no timeout. + WriteByteTimeout time.Duration + // MaxUploadBufferPerConnection is the size of the initial flow // control window for each connections. The HTTP/2 spec does not // allow this to be smaller than 65535 or larger than 2^32-1. @@ -4019,57 +4258,6 @@ func (s *http2Server) afterFunc(d time.Duration, f func()) http2timer { return http2timeTimer{time.AfterFunc(d, f)} } -func (s *http2Server) initialConnRecvWindowSize() int32 { - if s.MaxUploadBufferPerConnection >= http2initialWindowSize { - return s.MaxUploadBufferPerConnection - } - return 1 << 20 -} - -func (s *http2Server) initialStreamRecvWindowSize() int32 { - if s.MaxUploadBufferPerStream > 0 { - return s.MaxUploadBufferPerStream - } - return 1 << 20 -} - -func (s *http2Server) maxReadFrameSize() uint32 { - if v := s.MaxReadFrameSize; v >= http2minMaxFrameSize && v <= http2maxFrameSize { - return v - } - return http2defaultMaxReadFrameSize -} - -func (s *http2Server) maxConcurrentStreams() uint32 { - if v := s.MaxConcurrentStreams; v > 0 { - return v - } - return http2defaultMaxStreams -} - -func (s *http2Server) maxDecoderHeaderTableSize() uint32 { - if v := s.MaxDecoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - -func (s *http2Server) maxEncoderHeaderTableSize() uint32 { - if v := s.MaxEncoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - -// maxQueuedControlFrames is the maximum number of control frames like -// SETTINGS, PING and RST_STREAM that will be queued for writing before -// the connection is closed to prevent memory exhaustion attacks. -func (s *http2Server) maxQueuedControlFrames() int { - // TODO: if anybody asks, add a Server field, and remember to define the - // behavior of negative values. - return http2maxQueuedControlFrames -} - type http2serverInternalState struct { mu sync.Mutex activeConns map[*http2serverConn]struct{} @@ -4166,7 +4354,7 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { if s.TLSNextProto == nil { s.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){} } - protoHandler := func(hs *Server, c *tls.Conn, h Handler) { + protoHandler := func(hs *Server, c net.Conn, h Handler, sawClientPreface bool) { if http2testHookOnConn != nil { http2testHookOnConn() } @@ -4183,12 +4371,31 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { ctx = bc.BaseContext() } conf.ServeConn(c, &http2ServeConnOpts{ - Context: ctx, - Handler: h, - BaseConfig: hs, + Context: ctx, + Handler: h, + BaseConfig: hs, + SawClientPreface: sawClientPreface, }) } - s.TLSNextProto[http2NextProtoTLS] = protoHandler + s.TLSNextProto[http2NextProtoTLS] = func(hs *Server, c *tls.Conn, h Handler) { + protoHandler(hs, c, h, false) + } + // The "unencrypted_http2" TLSNextProto key is used to pass off non-TLS HTTP/2 conns. + // + // A connection passed in this method has already had the HTTP/2 preface read from it. + s.TLSNextProto[http2nextProtoUnencryptedHTTP2] = func(hs *Server, c *tls.Conn, h Handler) { + nc, err := http2unencryptedNetConnFromTLSConn(c) + if err != nil { + if lg := hs.ErrorLog; lg != nil { + lg.Print(err) + } else { + log.Print(err) + } + go c.Close() + return + } + protoHandler(hs, nc, h, true) + } return nil } @@ -4270,13 +4477,15 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( baseCtx, cancel := http2serverConnBaseContext(c, opts) defer cancel() + http1srv := opts.baseConfig() + conf := http2configFromServer(http1srv, s) sc := &http2serverConn{ srv: s, - hs: opts.baseConfig(), + hs: http1srv, conn: c, baseCtx: baseCtx, remoteAddrStr: c.RemoteAddr().String(), - bw: http2newBufferedWriter(c), + bw: http2newBufferedWriter(s.group, c, conf.WriteByteTimeout), handler: opts.handler(), streams: make(map[uint32]*http2stream), readFrameCh: make(chan http2readFrameResult), @@ -4286,9 +4495,12 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( bodyReadCh: make(chan http2bodyReadMsg), // buffering doesn't matter either way doneServing: make(chan struct{}), clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value" - advMaxStreams: s.maxConcurrentStreams(), + advMaxStreams: conf.MaxConcurrentStreams, initialStreamSendWindowSize: http2initialWindowSize, + initialStreamRecvWindowSize: conf.MaxUploadBufferPerStream, maxFrameSize: http2initialMaxFrameSize, + pingTimeout: conf.PingTimeout, + countErrorFunc: conf.CountError, serveG: http2newGoroutineLock(), pushEnabled: true, sawClientPreface: opts.SawClientPreface, @@ -4321,15 +4533,15 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( sc.flow.add(http2initialWindowSize) sc.inflow.init(http2initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) - sc.hpackEncoder.SetMaxDynamicTableSizeLimit(s.maxEncoderHeaderTableSize()) + sc.hpackEncoder.SetMaxDynamicTableSizeLimit(conf.MaxEncoderHeaderTableSize) fr := http2NewFramer(sc.bw, c) - if s.CountError != nil { - fr.countError = s.CountError + if conf.CountError != nil { + fr.countError = conf.CountError } - fr.ReadMetaHeaders = hpack.NewDecoder(s.maxDecoderHeaderTableSize(), nil) + fr.ReadMetaHeaders = hpack.NewDecoder(conf.MaxDecoderHeaderTableSize, nil) fr.MaxHeaderListSize = sc.maxHeaderListSize() - fr.SetMaxReadFrameSize(s.maxReadFrameSize()) + fr.SetMaxReadFrameSize(conf.MaxReadFrameSize) sc.framer = fr if tc, ok := c.(http2connectionStater); ok { @@ -4362,7 +4574,7 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( // So for now, do nothing here again. } - if !s.PermitProhibitedCipherSuites && http2isBadCipher(sc.tlsState.CipherSuite) { + if !conf.PermitProhibitedCipherSuites && http2isBadCipher(sc.tlsState.CipherSuite) { // "Endpoints MAY choose to generate a connection error // (Section 5.4.1) of type INADEQUATE_SECURITY if one of // the prohibited cipher suites are negotiated." @@ -4399,7 +4611,7 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( opts.UpgradeRequest = nil } - sc.serve() + sc.serve(conf) } func http2serverConnBaseContext(c net.Conn, opts *http2ServeConnOpts) (ctx context.Context, cancel func()) { @@ -4439,6 +4651,7 @@ type http2serverConn struct { tlsState *tls.ConnectionState // shared by all handlers, like net/http remoteAddrStr string writeSched http2WriteScheduler + countErrorFunc func(errType string) // Everything following is owned by the serve loop; use serveG.check(): serveG http2goroutineLock // used to verify funcs are on serve() @@ -4458,6 +4671,7 @@ type http2serverConn struct { streams map[uint32]*http2stream unstartedHandlers []http2unstartedHandler initialStreamSendWindowSize int32 + initialStreamRecvWindowSize int32 maxFrameSize int32 peerMaxHeaderListSize uint32 // zero means unknown (default) canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case @@ -4468,9 +4682,14 @@ type http2serverConn struct { inGoAway bool // we've started to or sent GOAWAY inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop needToSendGoAway bool // we need to schedule a GOAWAY frame write + pingSent bool + sentPingData [8]byte goAwayCode http2ErrCode shutdownTimer http2timer // nil until used idleTimer http2timer // nil if unused + readIdleTimeout time.Duration + pingTimeout time.Duration + readIdleTimer http2timer // nil if unused // Owned by the writeFrameAsync goroutine: headerWriteBuf bytes.Buffer @@ -4485,11 +4704,7 @@ func (sc *http2serverConn) maxHeaderListSize() uint32 { if n <= 0 { n = DefaultMaxHeaderBytes } - // http2's count is in a slightly different unit and includes 32 bytes per pair. - // So, take the net/http.Server value and pad it up a bit, assuming 10 headers. - const perFieldOverhead = 32 // per http2 spec - const typicalHeaders = 10 // conservative - return uint32(n + typicalHeaders*perFieldOverhead) + return uint32(http2adjustHTTP1MaxHeaderSize(int64(n))) } func (sc *http2serverConn) curOpenStreams() uint32 { @@ -4756,7 +4971,7 @@ func (sc *http2serverConn) notePanic() { } } -func (sc *http2serverConn) serve() { +func (sc *http2serverConn) serve(conf http2http2Config) { sc.serveG.check() defer sc.notePanic() defer sc.conn.Close() @@ -4768,20 +4983,24 @@ func (sc *http2serverConn) serve() { sc.vlogf("http2: server connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) } + settings := http2writeSettings{ + {http2SettingMaxFrameSize, conf.MaxReadFrameSize}, + {http2SettingMaxConcurrentStreams, sc.advMaxStreams}, + {http2SettingMaxHeaderListSize, sc.maxHeaderListSize()}, + {http2SettingHeaderTableSize, conf.MaxDecoderHeaderTableSize}, + {http2SettingInitialWindowSize, uint32(sc.initialStreamRecvWindowSize)}, + } + if !http2disableExtendedConnectProtocol { + settings = append(settings, http2Setting{http2SettingEnableConnectProtocol, 1}) + } sc.writeFrame(http2FrameWriteRequest{ - write: http2writeSettings{ - {http2SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, - {http2SettingMaxConcurrentStreams, sc.advMaxStreams}, - {http2SettingMaxHeaderListSize, sc.maxHeaderListSize()}, - {http2SettingHeaderTableSize, sc.srv.maxDecoderHeaderTableSize()}, - {http2SettingInitialWindowSize, uint32(sc.srv.initialStreamRecvWindowSize())}, - }, + write: settings, }) sc.unackedSettings++ // Each connection starts with initialWindowSize inflow tokens. // If a higher value is configured, we add more tokens. - if diff := sc.srv.initialConnRecvWindowSize() - http2initialWindowSize; diff > 0 { + if diff := conf.MaxUploadBufferPerConnection - http2initialWindowSize; diff > 0 { sc.sendWindowUpdate(nil, int(diff)) } @@ -4801,11 +5020,18 @@ func (sc *http2serverConn) serve() { defer sc.idleTimer.Stop() } + if conf.SendPingTimeout > 0 { + sc.readIdleTimeout = conf.SendPingTimeout + sc.readIdleTimer = sc.srv.afterFunc(conf.SendPingTimeout, sc.onReadIdleTimer) + defer sc.readIdleTimer.Stop() + } + go sc.readFrames() // closed by defer sc.conn.Close above settingsTimer := sc.srv.afterFunc(http2firstSettingsTimeout, sc.onSettingsTimer) defer settingsTimer.Stop() + lastFrameTime := sc.srv.now() loopNum := 0 for { loopNum++ @@ -4819,6 +5045,7 @@ func (sc *http2serverConn) serve() { case res := <-sc.wroteFrameCh: sc.wroteFrame(res) case res := <-sc.readFrameCh: + lastFrameTime = sc.srv.now() // Process any written frames before reading new frames from the client since a // written frame could have triggered a new stream to be started. if sc.writingFrameAsync { @@ -4850,6 +5077,8 @@ func (sc *http2serverConn) serve() { case http2idleTimerMsg: sc.vlogf("connection is idle") sc.goAway(http2ErrCodeNo) + case http2readIdleTimerMsg: + sc.handlePingTimer(lastFrameTime) case http2shutdownTimerMsg: sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) return @@ -4872,7 +5101,7 @@ func (sc *http2serverConn) serve() { // If the peer is causing us to generate a lot of control frames, // but not reading them from us, assume they are trying to make us // run out of memory. - if sc.queuedControlFrames > sc.srv.maxQueuedControlFrames() { + if sc.queuedControlFrames > http2maxQueuedControlFrames { sc.vlogf("http2: too many control frames in send queue, closing connection") return } @@ -4888,12 +5117,39 @@ func (sc *http2serverConn) serve() { } } +func (sc *http2serverConn) handlePingTimer(lastFrameReadTime time.Time) { + if sc.pingSent { + sc.vlogf("timeout waiting for PING response") + sc.conn.Close() + return + } + + pingAt := lastFrameReadTime.Add(sc.readIdleTimeout) + now := sc.srv.now() + if pingAt.After(now) { + // We received frames since arming the ping timer. + // Reset it for the next possible timeout. + sc.readIdleTimer.Reset(pingAt.Sub(now)) + return + } + + sc.pingSent = true + // Ignore crypto/rand.Read errors: It generally can't fail, and worse case if it does + // is we send a PING frame containing 0s. + _, _ = rand.Read(sc.sentPingData[:]) + sc.writeFrame(http2FrameWriteRequest{ + write: &http2writePing{data: sc.sentPingData}, + }) + sc.readIdleTimer.Reset(sc.pingTimeout) +} + type http2serverMessage int // Message values sent to serveMsgCh. var ( http2settingsTimerMsg = new(http2serverMessage) http2idleTimerMsg = new(http2serverMessage) + http2readIdleTimerMsg = new(http2serverMessage) http2shutdownTimerMsg = new(http2serverMessage) http2gracefulShutdownMsg = new(http2serverMessage) http2handlerDoneMsg = new(http2serverMessage) @@ -4903,6 +5159,8 @@ func (sc *http2serverConn) onSettingsTimer() { sc.sendServeMsg(http2settingsTime func (sc *http2serverConn) onIdleTimer() { sc.sendServeMsg(http2idleTimerMsg) } +func (sc *http2serverConn) onReadIdleTimer() { sc.sendServeMsg(http2readIdleTimerMsg) } + func (sc *http2serverConn) onShutdownTimer() { sc.sendServeMsg(http2shutdownTimerMsg) } func (sc *http2serverConn) sendServeMsg(msg interface{}) { @@ -5155,6 +5413,10 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { sc.writingFrame = false sc.writingFrameAsync = false + if res.err != nil { + sc.conn.Close() + } + wr := res.wr if http2writeEndsStream(wr.write) { @@ -5429,6 +5691,11 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { func (sc *http2serverConn) processPing(f *http2PingFrame) error { sc.serveG.check() if f.IsAck() { + if sc.pingSent && sc.sentPingData == f.Data { + // This is a response to a PING we sent. + sc.pingSent = false + sc.readIdleTimer.Reset(sc.readIdleTimeout) + } // 6.7 PING: " An endpoint MUST NOT respond to PING frames // containing this flag." return nil @@ -5592,6 +5859,9 @@ func (sc *http2serverConn) processSetting(s http2Setting) error { sc.maxFrameSize = int32(s.Val) // the maximum valid s.Val is < 2^31 case http2SettingMaxHeaderListSize: sc.peerMaxHeaderListSize = s.Val + case http2SettingEnableConnectProtocol: + // Receipt of this parameter by a server does not + // have any impact default: // Unknown setting: "An endpoint that receives a SETTINGS // frame with any unknown or unsupported identifier MUST @@ -5995,7 +6265,7 @@ func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState st.cw.Init() st.flow.conn = &sc.flow // link to conn-level counter st.flow.add(sc.initialStreamSendWindowSize) - st.inflow.init(sc.srv.initialStreamRecvWindowSize()) + st.inflow.init(sc.initialStreamRecvWindowSize) if sc.hs.WriteTimeout > 0 { st.writeDeadline = sc.srv.afterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) } @@ -6022,11 +6292,17 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead scheme: f.PseudoValue("scheme"), authority: f.PseudoValue("authority"), path: f.PseudoValue("path"), + protocol: f.PseudoValue("protocol"), + } + + // extended connect is disabled, so we should not see :protocol + if http2disableExtendedConnectProtocol && rp.protocol != "" { + return nil, nil, sc.countError("bad_connect", http2streamError(f.StreamID, http2ErrCodeProtocol)) } isConnect := rp.method == "CONNECT" if isConnect { - if rp.path != "" || rp.scheme != "" || rp.authority == "" { + if rp.protocol == "" && (rp.path != "" || rp.scheme != "" || rp.authority == "") { return nil, nil, sc.countError("bad_connect", http2streamError(f.StreamID, http2ErrCodeProtocol)) } } else if rp.method == "" || rp.path == "" || (rp.scheme != "https" && rp.scheme != "http") { @@ -6050,6 +6326,9 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead if rp.authority == "" { rp.authority = rp.header.Get("Host") } + if rp.protocol != "" { + rp.header.Set(":protocol", rp.protocol) + } rw, req, err := sc.newWriterAndRequestNoBody(st, rp) if err != nil { @@ -6076,6 +6355,7 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead type http2requestParam struct { method string scheme, authority, path string + protocol string header Header } @@ -6117,7 +6397,7 @@ func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2re var url_ *url.URL var requestURI string - if rp.method == "CONNECT" { + if rp.method == "CONNECT" && rp.protocol == "" { url_ = &url.URL{Host: rp.authority} requestURI = rp.authority // mimic HTTP/1 server behavior } else { @@ -6690,6 +6970,11 @@ func (w *http2responseWriter) SetWriteDeadline(deadline time.Time) error { return nil } +func (w *http2responseWriter) EnableFullDuplex() error { + // We always support full duplex responses, so this is a no-op. + return nil +} + func (w *http2responseWriter) Flush() { w.FlushError() } @@ -7136,7 +7421,7 @@ func (sc *http2serverConn) countError(name string, err error) error { if sc == nil || sc.srv == nil { return err } - f := sc.srv.CountError + f := sc.countErrorFunc if f == nil { return err } @@ -7339,6 +7624,20 @@ func (t *http2Transport) markNewGoroutine() { } } +func (t *http2Transport) now() time.Time { + if t != nil && t.http2transportTestHooks != nil { + return t.http2transportTestHooks.group.Now() + } + return time.Now() +} + +func (t *http2Transport) timeSince(when time.Time) time.Duration { + if t != nil && t.http2transportTestHooks != nil { + return t.now().Sub(when) + } + return time.Since(when) +} + // newTimer creates a new time.Timer, or a synthetic timer in tests. func (t *http2Transport) newTimer(d time.Duration) http2timer { if t.http2transportTestHooks != nil { @@ -7363,40 +7662,26 @@ func (t *http2Transport) contextWithTimeout(ctx context.Context, d time.Duration } func (t *http2Transport) maxHeaderListSize() uint32 { - if t.MaxHeaderListSize == 0 { + n := int64(t.MaxHeaderListSize) + if t.t1 != nil && t.t1.MaxResponseHeaderBytes != 0 { + n = t.t1.MaxResponseHeaderBytes + if n > 0 { + n = http2adjustHTTP1MaxHeaderSize(n) + } + } + if n <= 0 { return 10 << 20 } - if t.MaxHeaderListSize == 0xffffffff { + if n >= 0xffffffff { return 0 } - return t.MaxHeaderListSize -} - -func (t *http2Transport) maxFrameReadSize() uint32 { - if t.MaxReadFrameSize == 0 { - return 0 // use the default provided by the peer - } - if t.MaxReadFrameSize < http2minMaxFrameSize { - return http2minMaxFrameSize - } - if t.MaxReadFrameSize > http2maxFrameSize { - return http2maxFrameSize - } - return t.MaxReadFrameSize + return uint32(n) } func (t *http2Transport) disableCompression() bool { return t.DisableCompression || (t.t1 != nil && t.t1.DisableCompression) } -func (t *http2Transport) pingTimeout() time.Duration { - if t.PingTimeout == 0 { - return 15 * time.Second - } - return t.PingTimeout - -} - // ConfigureTransport configures a net/http HTTP/1 Transport to use HTTP/2. // It returns an error if t1 has already been HTTP/2-enabled. // @@ -7432,8 +7717,8 @@ func http2configureTransports(t1 *Transport) (*http2Transport, error) { if !http2strSliceContains(t1.TLSClientConfig.NextProtos, "http/1.1") { t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1") } - upgradeFn := func(authority string, c *tls.Conn) RoundTripper { - addr := http2authorityAddr("https", authority) + upgradeFn := func(scheme, authority string, c net.Conn) RoundTripper { + addr := http2authorityAddr(scheme, authority) if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil { go c.Close() return http2erringRoundTripper{err} @@ -7444,18 +7729,37 @@ func http2configureTransports(t1 *Transport) (*http2Transport, error) { // was unknown) go c.Close() } + if scheme == "http" { + return (*http2unencryptedTransport)(t2) + } return t2 } - if m := t1.TLSNextProto; len(m) == 0 { - t1.TLSNextProto = map[string]func(string, *tls.Conn) RoundTripper{ - "h2": upgradeFn, + if t1.TLSNextProto == nil { + t1.TLSNextProto = make(map[string]func(string, *tls.Conn) RoundTripper) + } + t1.TLSNextProto[http2NextProtoTLS] = func(authority string, c *tls.Conn) RoundTripper { + return upgradeFn("https", authority, c) + } + // The "unencrypted_http2" TLSNextProto key is used to pass off non-TLS HTTP/2 conns. + t1.TLSNextProto[http2nextProtoUnencryptedHTTP2] = func(authority string, c *tls.Conn) RoundTripper { + nc, err := http2unencryptedNetConnFromTLSConn(c) + if err != nil { + go c.Close() + return http2erringRoundTripper{err} } - } else { - m["h2"] = upgradeFn + return upgradeFn("http", authority, nc) } return t2, nil } +// unencryptedTransport is a Transport with a RoundTrip method that +// always permits http:// URLs. +type http2unencryptedTransport http2Transport + +func (t *http2unencryptedTransport) RoundTrip(req *Request) (*Response, error) { + return (*http2Transport)(t).RoundTripOpt(req, http2RoundTripOpt{allowHTTP: true}) +} + func (t *http2Transport) connPool() http2ClientConnPool { t.connPoolOnce.Do(t.initConnPool) return t.connPoolOrDef @@ -7475,7 +7779,7 @@ type http2ClientConn struct { t *http2Transport tconn net.Conn // usually *tls.Conn, except specialized impls tlsState *tls.ConnectionState // nil only for specialized impls - reused uint32 // whether conn is being reused; atomic + atomicReused uint32 // whether conn is being reused; atomic singleUse bool // whether being used for a single http.Request getConnCalled bool // used by clientConnPool @@ -7486,31 +7790,54 @@ type http2ClientConn struct { idleTimeout time.Duration // or 0 for never idleTimer http2timer - mu sync.Mutex // guards following - cond *sync.Cond // hold mu; broadcast on flow/closed changes - flow http2outflow // our conn-level flow control quota (cs.outflow is per stream) - inflow http2inflow // peer's conn-level flow control - doNotReuse bool // whether conn is marked to not be reused for any future requests - closing bool - closed bool - seenSettings bool // true if we've seen a settings frame, false otherwise - wantSettingsAck bool // we sent a SETTINGS frame and haven't heard back - goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received - goAwayDebug string // goAway frame's debug data, retained as a string - streams map[uint32]*http2clientStream // client-initiated - streamsReserved int // incr by ReserveNewRequest; decr on RoundTrip - nextStreamID uint32 - pendingRequests int // requests blocked and waiting to be sent because len(streams) == maxConcurrentStreams - pings map[[8]byte]chan struct{} // in flight ping data to notification channel - br *bufio.Reader - lastActive time.Time - lastIdle time.Time // time last idle + mu sync.Mutex // guards following + cond *sync.Cond // hold mu; broadcast on flow/closed changes + flow http2outflow // our conn-level flow control quota (cs.outflow is per stream) + inflow http2inflow // peer's conn-level flow control + doNotReuse bool // whether conn is marked to not be reused for any future requests + closing bool + closed bool + seenSettings bool // true if we've seen a settings frame, false otherwise + seenSettingsChan chan struct{} // closed when seenSettings is true or frame reading fails + wantSettingsAck bool // we sent a SETTINGS frame and haven't heard back + goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received + goAwayDebug string // goAway frame's debug data, retained as a string + streams map[uint32]*http2clientStream // client-initiated + streamsReserved int // incr by ReserveNewRequest; decr on RoundTrip + nextStreamID uint32 + pendingRequests int // requests blocked and waiting to be sent because len(streams) == maxConcurrentStreams + pings map[[8]byte]chan struct{} // in flight ping data to notification channel + br *bufio.Reader + lastActive time.Time + lastIdle time.Time // time last idle // Settings from peer: (also guarded by wmu) - maxFrameSize uint32 - maxConcurrentStreams uint32 - peerMaxHeaderListSize uint64 - peerMaxHeaderTableSize uint32 - initialWindowSize uint32 + maxFrameSize uint32 + maxConcurrentStreams uint32 + peerMaxHeaderListSize uint64 + peerMaxHeaderTableSize uint32 + initialWindowSize uint32 + initialStreamRecvWindowSize int32 + readIdleTimeout time.Duration + pingTimeout time.Duration + extendedConnectAllowed bool + + // rstStreamPingsBlocked works around an unfortunate gRPC behavior. + // gRPC strictly limits the number of PING frames that it will receive. + // The default is two pings per two hours, but the limit resets every time + // the gRPC endpoint sends a HEADERS or DATA frame. See golang/go#70575. + // + // rstStreamPingsBlocked is set after receiving a response to a PING frame + // bundled with an RST_STREAM (see pendingResets below), and cleared after + // receiving a HEADERS or DATA frame. + rstStreamPingsBlocked bool + + // pendingResets is the number of RST_STREAM frames we have sent to the peer, + // without confirming that the peer has received them. When we send a RST_STREAM, + // we bundle it with a PING frame, unless a PING is already in flight. We count + // the reset stream against the connection's concurrency limit until we get + // a PING response. This limits the number of requests we'll try to send to a + // completely unresponsive connection. + pendingResets int // reqHeaderMu is a 1-element semaphore channel controlling access to sending new requests. // Write to reqHeaderMu to lock it, read from it to unlock. @@ -7568,12 +7895,12 @@ type http2clientStream struct { sentHeaders bool // owned by clientConnReadLoop: - firstByte bool // got the first response byte - pastHeaders bool // got first MetaHeadersFrame (actual headers) - pastTrailers bool // got optional second MetaHeadersFrame (trailers) - num1xx uint8 // number of 1xx responses seen - readClosed bool // peer sent an END_STREAM flag - readAborted bool // read loop reset the stream + firstByte bool // got the first response byte + pastHeaders bool // got first MetaHeadersFrame (actual headers) + pastTrailers bool // got optional second MetaHeadersFrame (trailers) + readClosed bool // peer sent an END_STREAM flag + readAborted bool // read loop reset the stream + totalHeaderSize int64 // total size of 1xx headers seen trailer Header // accumulated trailers resTrailer *Header // client's Response.Trailer @@ -7635,6 +7962,7 @@ func (cs *http2clientStream) closeReqBodyLocked() { } type http2stickyErrWriter struct { + group http2synctestGroupInterface conn net.Conn timeout time.Duration err *error @@ -7644,22 +7972,9 @@ func (sew http2stickyErrWriter) Write(p []byte) (n int, err error) { if *sew.err != nil { return 0, *sew.err } - for { - if sew.timeout != 0 { - sew.conn.SetWriteDeadline(time.Now().Add(sew.timeout)) - } - nn, err := sew.conn.Write(p[n:]) - n += nn - if n < len(p) && nn > 0 && errors.Is(err, os.ErrDeadlineExceeded) { - // Keep extending the deadline so long as we're making progress. - continue - } - if sew.timeout != 0 { - sew.conn.SetWriteDeadline(time.Time{}) - } - *sew.err = err - return n, err - } + n, err = http2writeWithByteTimeout(sew.group, sew.conn, sew.timeout, p) + *sew.err = err + return n, err } // noCachedConnError is the concrete type of ErrNoCachedConn, which @@ -7691,6 +8006,8 @@ type http2RoundTripOpt struct { // no cached connection is available, RoundTripOpt // will return ErrNoCachedConn. OnlyCachedConn bool + + allowHTTP bool // allow http:// URLs } func (t *http2Transport) RoundTrip(req *Request) (*Response, error) { @@ -7723,7 +8040,14 @@ func http2authorityAddr(scheme string, authority string) (addr string) { // RoundTripOpt is like RoundTrip, but takes options. func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Response, error) { - if !(req.URL.Scheme == "https" || (req.URL.Scheme == "http" && t.AllowHTTP)) { + switch req.URL.Scheme { + case "https": + // Always okay. + case "http": + if !t.AllowHTTP && !opt.allowHTTP { + return nil, errors.New("http2: unencrypted HTTP/2 not enabled") + } + default: return nil, errors.New("http2: unsupported scheme") } @@ -7734,7 +8058,7 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) return nil, err } - reused := !atomic.CompareAndSwapUint32(&cc.reused, 0, 1) + reused := !atomic.CompareAndSwapUint32(&cc.atomicReused, 0, 1) http2traceGotConn(req, cc, reused) res, err := cc.RoundTrip(req) if err != nil && retry <= 6 { @@ -7759,6 +8083,22 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res } } } + if err == http2errClientConnNotEstablished { + // This ClientConn was created recently, + // this is the first request to use it, + // and the connection is closed and not usable. + // + // In this state, cc.idleTimer will remove the conn from the pool + // when it fires. Stop the timer and remove it here so future requests + // won't try to use this connection. + // + // If the timer has already fired and we're racing it, the redundant + // call to MarkDead is harmless. + if cc.idleTimer != nil { + cc.idleTimer.Stop() + } + t.connPool().MarkDead(cc) + } if err != nil { t.vlogf("RoundTrip failure: %v", err) return nil, err @@ -7777,9 +8117,10 @@ func (t *http2Transport) CloseIdleConnections() { } var ( - http2errClientConnClosed = errors.New("http2: client conn is closed") - http2errClientConnUnusable = errors.New("http2: client conn not usable") - http2errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") + http2errClientConnClosed = errors.New("http2: client conn is closed") + http2errClientConnUnusable = errors.New("http2: client conn not usable") + http2errClientConnNotEstablished = errors.New("http2: client conn could not be established") + http2errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") ) // shouldRetryRequest is called by RoundTrip when a request fails to get @@ -7895,44 +8236,38 @@ func (t *http2Transport) expectContinueTimeout() time.Duration { return t.t1.ExpectContinueTimeout } -func (t *http2Transport) maxDecoderHeaderTableSize() uint32 { - if v := t.MaxDecoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - -func (t *http2Transport) maxEncoderHeaderTableSize() uint32 { - if v := t.MaxEncoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { return t.newClientConn(c, t.disableKeepAlives()) } func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2ClientConn, error) { + conf := http2configFromTransport(t) cc := &http2ClientConn{ - t: t, - tconn: c, - readerDone: make(chan struct{}), - nextStreamID: 1, - maxFrameSize: 16 << 10, // spec default - initialWindowSize: 65535, // spec default - maxConcurrentStreams: http2initialMaxConcurrentStreams, // "infinite", per spec. Use a smaller value until we have received server settings. - peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead. - streams: make(map[uint32]*http2clientStream), - singleUse: singleUse, - wantSettingsAck: true, - pings: make(map[[8]byte]chan struct{}), - reqHeaderMu: make(chan struct{}, 1), + t: t, + tconn: c, + readerDone: make(chan struct{}), + nextStreamID: 1, + maxFrameSize: 16 << 10, // spec default + initialWindowSize: 65535, // spec default + initialStreamRecvWindowSize: conf.MaxUploadBufferPerStream, + maxConcurrentStreams: http2initialMaxConcurrentStreams, // "infinite", per spec. Use a smaller value until we have received server settings. + peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead. + streams: make(map[uint32]*http2clientStream), + singleUse: singleUse, + seenSettingsChan: make(chan struct{}), + wantSettingsAck: true, + readIdleTimeout: conf.SendPingTimeout, + pingTimeout: conf.PingTimeout, + pings: make(map[[8]byte]chan struct{}), + reqHeaderMu: make(chan struct{}, 1), + lastActive: t.now(), } + var group http2synctestGroupInterface if t.http2transportTestHooks != nil { t.markNewGoroutine() t.http2transportTestHooks.newclientconn(cc) c = cc.tconn + group = t.group } if http2VerboseLogs { t.vlogf("http2: Transport creating client conn %p to %v", cc, c.RemoteAddr()) @@ -7944,30 +8279,25 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client // TODO: adjust this writer size to account for frame size + // MTU + crypto/tls record padding. cc.bw = bufio.NewWriter(http2stickyErrWriter{ + group: group, conn: c, - timeout: t.WriteByteTimeout, + timeout: conf.WriteByteTimeout, err: &cc.werr, }) cc.br = bufio.NewReader(c) cc.fr = http2NewFramer(cc.bw, cc.br) - if t.maxFrameReadSize() != 0 { - cc.fr.SetMaxReadFrameSize(t.maxFrameReadSize()) - } + cc.fr.SetMaxReadFrameSize(conf.MaxReadFrameSize) if t.CountError != nil { cc.fr.countError = t.CountError } - maxHeaderTableSize := t.maxDecoderHeaderTableSize() + maxHeaderTableSize := conf.MaxDecoderHeaderTableSize cc.fr.ReadMetaHeaders = hpack.NewDecoder(maxHeaderTableSize, nil) cc.fr.MaxHeaderListSize = t.maxHeaderListSize() cc.henc = hpack.NewEncoder(&cc.hbuf) - cc.henc.SetMaxDynamicTableSizeLimit(t.maxEncoderHeaderTableSize()) + cc.henc.SetMaxDynamicTableSizeLimit(conf.MaxEncoderHeaderTableSize) cc.peerMaxHeaderTableSize = http2initialHeaderTableSize - if t.AllowHTTP { - cc.nextStreamID = 3 - } - if cs, ok := c.(http2connectionStater); ok { state := cs.ConnectionState() cc.tlsState = &state @@ -7975,11 +8305,9 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client initialSettings := []http2Setting{ {ID: http2SettingEnablePush, Val: 0}, - {ID: http2SettingInitialWindowSize, Val: http2transportDefaultStreamFlow}, - } - if max := t.maxFrameReadSize(); max != 0 { - initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxFrameSize, Val: max}) + {ID: http2SettingInitialWindowSize, Val: uint32(cc.initialStreamRecvWindowSize)}, } + initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxFrameSize, Val: conf.MaxReadFrameSize}) if max := t.maxHeaderListSize(); max != 0 { initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxHeaderListSize, Val: max}) } @@ -7989,8 +8317,8 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client cc.bw.Write(http2clientPreface) cc.fr.WriteSettings(initialSettings...) - cc.fr.WriteWindowUpdate(0, http2transportDefaultConnFlow) - cc.inflow.init(http2transportDefaultConnFlow + http2initialWindowSize) + cc.fr.WriteWindowUpdate(0, uint32(conf.MaxUploadBufferPerConnection)) + cc.inflow.init(conf.MaxUploadBufferPerConnection + http2initialWindowSize) cc.bw.Flush() if cc.werr != nil { cc.Close() @@ -8008,7 +8336,7 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client } func (cc *http2ClientConn) healthCheck() { - pingTimeout := cc.t.pingTimeout() + pingTimeout := cc.pingTimeout // We don't need to periodically ping in the health check, because the readLoop of ClientConn will // trigger the healthCheck again if there is no frame received. ctx, cancel := cc.t.contextWithTimeout(context.Background(), pingTimeout) @@ -8136,7 +8464,7 @@ func (cc *http2ClientConn) State() http2ClientConnState { return http2ClientConnState{ Closed: cc.closed, Closing: cc.closing || cc.singleUse || cc.doNotReuse || cc.goAway != nil, - StreamsActive: len(cc.streams), + StreamsActive: len(cc.streams) + cc.pendingResets, StreamsReserved: cc.streamsReserved, StreamsPending: cc.pendingRequests, LastIdle: cc.lastIdle, @@ -8168,16 +8496,38 @@ func (cc *http2ClientConn) idleStateLocked() (st http2clientConnIdleState) { // writing it. maxConcurrentOkay = true } else { - maxConcurrentOkay = int64(len(cc.streams)+cc.streamsReserved+1) <= int64(cc.maxConcurrentStreams) + // We can take a new request if the total of + // - active streams; + // - reservation slots for new streams; and + // - streams for which we have sent a RST_STREAM and a PING, + // but received no subsequent frame + // is less than the concurrency limit. + maxConcurrentOkay = cc.currentRequestCountLocked() < int(cc.maxConcurrentStreams) } st.canTakeNewRequest = cc.goAway == nil && !cc.closed && !cc.closing && maxConcurrentOkay && !cc.doNotReuse && int64(cc.nextStreamID)+2*int64(cc.pendingRequests) < math.MaxInt32 && !cc.tooIdleLocked() + + // If this connection has never been used for a request and is closed, + // then let it take a request (which will fail). + // + // This avoids a situation where an error early in a connection's lifetime + // goes unreported. + if cc.nextStreamID == 1 && cc.streamsReserved == 0 && cc.closed { + st.canTakeNewRequest = true + } + return } +// currentRequestCountLocked reports the number of concurrency slots currently in use, +// including active streams, reserved slots, and reset streams waiting for acknowledgement. +func (cc *http2ClientConn) currentRequestCountLocked() int { + return len(cc.streams) + cc.streamsReserved + cc.pendingResets +} + func (cc *http2ClientConn) canTakeNewRequestLocked() bool { st := cc.idleStateLocked() return st.canTakeNewRequest @@ -8190,7 +8540,7 @@ func (cc *http2ClientConn) tooIdleLocked() bool { // times are compared based on their wall time. We don't want // to reuse a connection that's been sitting idle during // VM/laptop suspend if monotonic time was also frozen. - return cc.idleTimeout != 0 && !cc.lastIdle.IsZero() && time.Since(cc.lastIdle.Round(0)) > cc.idleTimeout + return cc.idleTimeout != 0 && !cc.lastIdle.IsZero() && cc.t.timeSince(cc.lastIdle.Round(0)) > cc.idleTimeout } // onIdleTimeout is called from a time.AfterFunc goroutine. It will @@ -8552,6 +8902,8 @@ func (cs *http2clientStream) doRequest(req *Request, streamf func(*http2clientSt cs.cleanupWriteRequest(err) } +var http2errExtendedConnectNotSupported = errors.New("net/http: extended connect not supported by peer") + // writeRequest sends a request. // // It returns nil after the request is written, the response read, @@ -8567,12 +8919,31 @@ func (cs *http2clientStream) writeRequest(req *Request, streamf func(*http2clien return err } + // wait for setting frames to be received, a server can change this value later, + // but we just wait for the first settings frame + var isExtendedConnect bool + if req.Method == "CONNECT" && req.Header.Get(":protocol") != "" { + isExtendedConnect = true + } + // Acquire the new-request lock by writing to reqHeaderMu. // This lock guards the critical section covering allocating a new stream ID // (requires mu) and creating the stream (requires wmu). if cc.reqHeaderMu == nil { panic("RoundTrip on uninitialized ClientConn") // for tests } + if isExtendedConnect { + select { + case <-cs.reqCancel: + return http2errRequestCanceled + case <-ctx.Done(): + return ctx.Err() + case <-cc.seenSettingsChan: + if !cc.extendedConnectAllowed { + return http2errExtendedConnectNotSupported + } + } + } select { case cc.reqHeaderMu <- struct{}{}: case <-cs.reqCancel: @@ -8754,6 +9125,7 @@ func (cs *http2clientStream) cleanupWriteRequest(err error) { cs.reqBodyClosed = make(chan struct{}) } bodyClosed := cs.reqBodyClosed + closeOnIdle := cc.singleUse || cc.doNotReuse || cc.t.disableKeepAlives() || cc.goAway != nil cc.mu.Unlock() if mustCloseBody { cs.reqBody.Close() @@ -8778,16 +9150,44 @@ func (cs *http2clientStream) cleanupWriteRequest(err error) { if cs.sentHeaders { if se, ok := err.(http2StreamError); ok { if se.Cause != http2errFromPeer { - cc.writeStreamReset(cs.ID, se.Code, err) + cc.writeStreamReset(cs.ID, se.Code, false, err) } } else { - cc.writeStreamReset(cs.ID, http2ErrCodeCancel, err) + // We're cancelling an in-flight request. + // + // This could be due to the server becoming unresponsive. + // To avoid sending too many requests on a dead connection, + // we let the request continue to consume a concurrency slot + // until we can confirm the server is still responding. + // We do this by sending a PING frame along with the RST_STREAM + // (unless a ping is already in flight). + // + // For simplicity, we don't bother tracking the PING payload: + // We reset cc.pendingResets any time we receive a PING ACK. + // + // We skip this if the conn is going to be closed on idle, + // because it's short lived and will probably be closed before + // we get the ping response. + ping := false + if !closeOnIdle { + cc.mu.Lock() + // rstStreamPingsBlocked works around a gRPC behavior: + // see comment on the field for details. + if !cc.rstStreamPingsBlocked { + if cc.pendingResets == 0 { + ping = true + } + cc.pendingResets++ + } + cc.mu.Unlock() + } + cc.writeStreamReset(cs.ID, http2ErrCodeCancel, ping, err) } } cs.bufPipe.CloseWithError(err) // no-op if already closed } else { if cs.sentHeaders && !cs.sentEndStream { - cc.writeStreamReset(cs.ID, http2ErrCodeNo, nil) + cc.writeStreamReset(cs.ID, http2ErrCodeNo, false, nil) } cs.bufPipe.CloseWithError(http2errRequestCanceled) } @@ -8809,12 +9209,17 @@ func (cs *http2clientStream) cleanupWriteRequest(err error) { // Must hold cc.mu. func (cc *http2ClientConn) awaitOpenSlotForStreamLocked(cs *http2clientStream) error { for { - cc.lastActive = time.Now() + if cc.closed && cc.nextStreamID == 1 && cc.streamsReserved == 0 { + // This is the very first request sent to this connection. + // Return a fatal error which aborts the retry loop. + return http2errClientConnNotEstablished + } + cc.lastActive = cc.t.now() if cc.closed || !cc.canTakeNewRequestLocked() { return http2errClientConnUnusable } cc.lastIdle = time.Time{} - if int64(len(cc.streams)) < int64(cc.maxConcurrentStreams) { + if cc.currentRequestCountLocked() < int(cc.maxConcurrentStreams) { return nil } cc.pendingRequests++ @@ -9087,7 +9492,7 @@ func (cs *http2clientStream) awaitFlowControl(maxBytes int) (taken int32, err er func http2validateHeaders(hdrs Header) string { for k, vv := range hdrs { - if !httpguts.ValidHeaderFieldName(k) { + if !httpguts.ValidHeaderFieldName(k) && k != ":protocol" { return fmt.Sprintf("name %q", k) } for _, v := range vv { @@ -9122,8 +9527,17 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail return nil, errors.New("http2: invalid Host header") } + // isNormalConnect is true if this is a non-extended CONNECT request. + isNormalConnect := false + protocol := req.Header.Get(":protocol") + if req.Method == "CONNECT" && protocol == "" { + isNormalConnect = true + } else if protocol != "" && req.Method != "CONNECT" { + return nil, errors.New("http2: invalid :protocol header in non-CONNECT request") + } + var path string - if req.Method != "CONNECT" { + if !isNormalConnect { path = req.URL.RequestURI() if !http2validPseudoPath(path) { orig := path @@ -9160,10 +9574,13 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail m = MethodGet } f(":method", m) - if req.Method != "CONNECT" { + if !isNormalConnect { f(":path", path) f(":scheme", req.URL.Scheme) } + if protocol != "" { + f(":protocol", protocol) + } if trailers != "" { f("trailer", trailers) } @@ -9220,6 +9637,9 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail } } continue + } else if k == ":protocol" { + // :protocol pseudo-header was already sent above. + continue } for _, v := range vv { @@ -9341,7 +9761,7 @@ type http2resAndError struct { func (cc *http2ClientConn) addStreamLocked(cs *http2clientStream) { cs.flow.add(int32(cc.initialWindowSize)) cs.flow.setConnFlow(&cc.flow) - cs.inflow.init(http2transportDefaultStreamFlow) + cs.inflow.init(cc.initialStreamRecvWindowSize) cs.ID = cc.nextStreamID cc.nextStreamID += 2 cc.streams[cs.ID] = cs @@ -9357,10 +9777,10 @@ func (cc *http2ClientConn) forgetStreamID(id uint32) { if len(cc.streams) != slen-1 { panic("forgetting unknown stream id") } - cc.lastActive = time.Now() + cc.lastActive = cc.t.now() if len(cc.streams) == 0 && cc.idleTimer != nil { cc.idleTimer.Reset(cc.idleTimeout) - cc.lastIdle = time.Now() + cc.lastIdle = cc.t.now() } // Wake up writeRequestBody via clientStream.awaitFlowControl and // wake up RoundTrip if there is a pending request. @@ -9420,7 +9840,6 @@ func http2isEOFOrNetReadError(err error) bool { func (rl *http2clientConnReadLoop) cleanup() { cc := rl.cc - cc.t.connPool().MarkDead(cc) defer cc.closeConn() defer close(cc.readerDone) @@ -9444,6 +9863,24 @@ func (rl *http2clientConnReadLoop) cleanup() { } cc.closed = true + // If the connection has never been used, and has been open for only a short time, + // leave it in the connection pool for a little while. + // + // This avoids a situation where new connections are constantly created, + // added to the pool, fail, and are removed from the pool, without any error + // being surfaced to the user. + const unusedWaitTime = 5 * time.Second + idleTime := cc.t.now().Sub(cc.lastActive) + if atomic.LoadUint32(&cc.atomicReused) == 0 && idleTime < unusedWaitTime { + cc.idleTimer = cc.t.afterFunc(unusedWaitTime-idleTime, func() { + cc.t.connPool().MarkDead(cc) + }) + } else { + cc.mu.Unlock() // avoid any deadlocks in MarkDead + cc.t.connPool().MarkDead(cc) + cc.mu.Lock() + } + for _, cs := range cc.streams { select { case <-cs.peerClosed: @@ -9487,7 +9924,7 @@ func (cc *http2ClientConn) countReadFrameError(err error) { func (rl *http2clientConnReadLoop) run() error { cc := rl.cc gotSettings := false - readIdleTimeout := cc.t.ReadIdleTimeout + readIdleTimeout := cc.readIdleTimeout var t http2timer if readIdleTimeout != 0 { t = cc.t.afterFunc(readIdleTimeout, cc.healthCheck) @@ -9501,7 +9938,7 @@ func (rl *http2clientConnReadLoop) run() error { cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err) } if se, ok := err.(http2StreamError); ok { - if cs := rl.streamByID(se.StreamID); cs != nil { + if cs := rl.streamByID(se.StreamID, http2notHeaderOrDataFrame); cs != nil { if se.Cause == nil { se.Cause = cc.fr.errDetail } @@ -9547,13 +9984,16 @@ func (rl *http2clientConnReadLoop) run() error { if http2VerboseLogs { cc.vlogf("http2: Transport conn %p received error from processing frame %v: %v", cc, http2summarizeFrame(f), err) } + if !cc.seenSettings { + close(cc.seenSettingsChan) + } return err } } } func (rl *http2clientConnReadLoop) processHeaders(f *http2MetaHeadersFrame) error { - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2headerOrDataFrame) if cs == nil { // We'd get here if we canceled a request while the // server had its response still in flight. So if this @@ -9671,15 +10111,34 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http if f.StreamEnded() { return nil, errors.New("1xx informational response with END_STREAM flag") } - cs.num1xx++ - const max1xxResponses = 5 // arbitrary bound on number of informational responses, same as net/http - if cs.num1xx > max1xxResponses { - return nil, errors.New("http2: too many 1xx informational responses") - } if fn := cs.get1xxTraceFunc(); fn != nil { + // If the 1xx response is being delivered to the user, + // then they're responsible for limiting the number + // of responses. if err := fn(statusCode, textproto.MIMEHeader(header)); err != nil { return nil, err } + } else { + // If the user didn't examine the 1xx response, then we + // limit the size of all 1xx headers. + // + // This differs a bit from the HTTP/1 implementation, which + // limits the size of all 1xx headers plus the final response. + // Use the larger limit of MaxHeaderListSize and + // net/http.Transport.MaxResponseHeaderBytes. + limit := int64(cs.cc.t.maxHeaderListSize()) + if t1 := cs.cc.t.t1; t1 != nil && t1.MaxResponseHeaderBytes > limit { + limit = t1.MaxResponseHeaderBytes + } + for _, h := range f.Fields { + cs.totalHeaderSize += int64(h.Size()) + } + if cs.totalHeaderSize > limit { + if http2VerboseLogs { + log.Printf("http2: 1xx informational responses too large") + } + return nil, errors.New("header list too large") + } } if statusCode == 100 { http2traceGot100Continue(cs.trace) @@ -9863,7 +10322,7 @@ func (b http2transportResponseBody) Close() error { func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { cc := rl.cc - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2headerOrDataFrame) data := f.Data() if cs == nil { cc.mu.Lock() @@ -9998,9 +10457,22 @@ func (rl *http2clientConnReadLoop) endStreamError(cs *http2clientStream, err err cs.abortStream(err) } -func (rl *http2clientConnReadLoop) streamByID(id uint32) *http2clientStream { +// Constants passed to streamByID for documentation purposes. +const ( + http2headerOrDataFrame = true + http2notHeaderOrDataFrame = false +) + +// streamByID returns the stream with the given id, or nil if no stream has that id. +// If headerOrData is true, it clears rst.StreamPingsBlocked. +func (rl *http2clientConnReadLoop) streamByID(id uint32, headerOrData bool) *http2clientStream { rl.cc.mu.Lock() defer rl.cc.mu.Unlock() + if headerOrData { + // Work around an unfortunate gRPC behavior. + // See comment on ClientConn.rstStreamPingsBlocked for details. + rl.cc.rstStreamPingsBlocked = false + } cs := rl.cc.streams[id] if cs != nil && !cs.readAborted { return cs @@ -10094,6 +10566,21 @@ func (rl *http2clientConnReadLoop) processSettingsNoWrite(f *http2SettingsFrame) case http2SettingHeaderTableSize: cc.henc.SetMaxDynamicTableSize(s.Val) cc.peerMaxHeaderTableSize = s.Val + case http2SettingEnableConnectProtocol: + if err := s.Valid(); err != nil { + return err + } + // If the peer wants to send us SETTINGS_ENABLE_CONNECT_PROTOCOL, + // we require that it do so in the first SETTINGS frame. + // + // When we attempt to use extended CONNECT, we wait for the first + // SETTINGS frame to see if the server supports it. If we let the + // server enable the feature with a later SETTINGS frame, then + // users will see inconsistent results depending on whether we've + // seen that frame or not. + if !cc.seenSettings { + cc.extendedConnectAllowed = s.Val == 1 + } default: cc.vlogf("Unhandled Setting: %v", s) } @@ -10111,6 +10598,7 @@ func (rl *http2clientConnReadLoop) processSettingsNoWrite(f *http2SettingsFrame) // connection can establish to our default. cc.maxConcurrentStreams = http2defaultMaxConcurrentStreams } + close(cc.seenSettingsChan) cc.seenSettings = true } @@ -10119,7 +10607,7 @@ func (rl *http2clientConnReadLoop) processSettingsNoWrite(f *http2SettingsFrame) func (rl *http2clientConnReadLoop) processWindowUpdate(f *http2WindowUpdateFrame) error { cc := rl.cc - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2notHeaderOrDataFrame) if f.StreamID != 0 && cs == nil { return nil } @@ -10148,7 +10636,7 @@ func (rl *http2clientConnReadLoop) processWindowUpdate(f *http2WindowUpdateFrame } func (rl *http2clientConnReadLoop) processResetStream(f *http2RSTStreamFrame) error { - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2notHeaderOrDataFrame) if cs == nil { // TODO: return error if server tries to RST_STREAM an idle stream return nil @@ -10223,6 +10711,12 @@ func (rl *http2clientConnReadLoop) processPing(f *http2PingFrame) error { close(c) delete(cc.pings, f.Data) } + if cc.pendingResets > 0 { + // See clientStream.cleanupWriteRequest. + cc.pendingResets = 0 + cc.rstStreamPingsBlocked = true + cc.cond.Broadcast() + } return nil } cc := rl.cc @@ -10245,13 +10739,20 @@ func (rl *http2clientConnReadLoop) processPushPromise(f *http2PushPromiseFrame) return http2ConnectionError(http2ErrCodeProtocol) } -func (cc *http2ClientConn) writeStreamReset(streamID uint32, code http2ErrCode, err error) { +// writeStreamReset sends a RST_STREAM frame. +// When ping is true, it also sends a PING frame with a random payload. +func (cc *http2ClientConn) writeStreamReset(streamID uint32, code http2ErrCode, ping bool, err error) { // TODO: map err to more interesting error codes, once the // HTTP community comes up with some. But currently for // RST_STREAM there's no equivalent to GOAWAY frame's debug // data, and the error codes are all pretty vague ("cancel"). cc.wmu.Lock() cc.fr.WriteRSTStream(streamID, code) + if ping { + var payload [8]byte + rand.Read(payload[:]) + cc.fr.WritePing(false, payload) + } cc.bw.Flush() cc.wmu.Unlock() } @@ -10408,7 +10909,7 @@ func http2traceGotConn(req *Request, cc *http2ClientConn, reused bool) { cc.mu.Lock() ci.WasIdle = len(cc.streams) == 0 && reused if ci.WasIdle && !cc.lastActive.IsZero() { - ci.IdleTime = time.Since(cc.lastActive) + ci.IdleTime = cc.t.timeSince(cc.lastActive) } cc.mu.Unlock() @@ -10476,6 +10977,27 @@ func (t *http2Transport) dialTLSWithContext(ctx context.Context, network, addr s return tlsCn, nil } +const http2nextProtoUnencryptedHTTP2 = "unencrypted_http2" + +// unencryptedNetConnFromTLSConn retrieves a net.Conn wrapped in a *tls.Conn. +// +// TLSNextProto functions accept a *tls.Conn. +// +// When passing an unencrypted HTTP/2 connection to a TLSNextProto function, +// we pass a *tls.Conn with an underlying net.Conn containing the unencrypted connection. +// To be extra careful about mistakes (accidentally dropping TLS encryption in a place +// where we want it), the tls.Conn contains a net.Conn with an UnencryptedNetConn method +// that returns the actual connection we want to use. +func http2unencryptedNetConnFromTLSConn(tc *tls.Conn) (net.Conn, error) { + conner, ok := tc.NetConn().(interface { + UnencryptedNetConn() net.Conn + }) + if !ok { + return nil, errors.New("http2: TLS conn unexpectedly found in unencrypted handoff") + } + return conner.UnencryptedNetConn(), nil +} + // writeFramer is implemented by any type that is used to write frames. type http2writeFramer interface { writeFrame(http2writeContext) error @@ -10592,6 +11114,18 @@ func (se http2StreamError) writeFrame(ctx http2writeContext) error { func (se http2StreamError) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } +type http2writePing struct { + data [8]byte +} + +func (w http2writePing) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WritePing(false, w.data) +} + +func (w http2writePing) staysWithinBuffer(max int) bool { + return http2frameHeaderLen+len(w.data) <= max +} + type http2writePingAck struct{ pf *http2PingFrame } func (w http2writePingAck) writeFrame(ctx http2writeContext) error { diff --git a/src/net/http/http.go b/src/net/http/http.go index 6e2259ad..32ff7e20 100644 --- a/src/net/http/http.go +++ b/src/net/http/http.go @@ -16,6 +16,67 @@ import ( "golang.org/x/net/http/httpguts" ) +// Protocols is a set of HTTP protocols. +// The zero value is an empty set of protocols. +// +// The supported protocols are: +// +// - HTTP1 is the HTTP/1.0 and HTTP/1.1 protocols. +// HTTP1 is supported on both unsecured TCP and secured TLS connections. +// +// - HTTP2 is the HTTP/2 protcol over a TLS connection. +// +// - UnencryptedHTTP2 is the HTTP/2 protocol over an unsecured TCP connection. +type Protocols struct { + bits uint8 +} + +const ( + protoHTTP1 = 1 << iota + protoHTTP2 + protoUnencryptedHTTP2 +) + +// HTTP1 reports whether p includes HTTP/1. +func (p Protocols) HTTP1() bool { return p.bits&protoHTTP1 != 0 } + +// SetHTTP1 adds or removes HTTP/1 from p. +func (p *Protocols) SetHTTP1(ok bool) { p.setBit(protoHTTP1, ok) } + +// HTTP2 reports whether p includes HTTP/2. +func (p Protocols) HTTP2() bool { return p.bits&protoHTTP2 != 0 } + +// SetHTTP2 adds or removes HTTP/2 from p. +func (p *Protocols) SetHTTP2(ok bool) { p.setBit(protoHTTP2, ok) } + +// UnencryptedHTTP2 reports whether p includes unencrypted HTTP/2. +func (p Protocols) UnencryptedHTTP2() bool { return p.bits&protoUnencryptedHTTP2 != 0 } + +// SetUnencryptedHTTP2 adds or removes unencrypted HTTP/2 from p. +func (p *Protocols) SetUnencryptedHTTP2(ok bool) { p.setBit(protoUnencryptedHTTP2, ok) } + +func (p *Protocols) setBit(bit uint8, ok bool) { + if ok { + p.bits |= bit + } else { + p.bits &^= bit + } +} + +func (p Protocols) String() string { + var s []string + if p.HTTP1() { + s = append(s, "HTTP1") + } + if p.HTTP2() { + s = append(s, "HTTP2") + } + if p.UnencryptedHTTP2() { + s = append(s, "UnencryptedHTTP2") + } + return "{" + strings.Join(s, ",") + "}" +} + // incomparable is a zero-width, non-comparable type. Adding it to a struct // makes that struct also non-comparable, and generally doesn't add // any size (as long as it's first). @@ -163,3 +224,68 @@ type Pusher interface { // is not supported on the underlying connection. Push(target string, opts *PushOptions) error } + +// HTTP2Config defines HTTP/2 configuration parameters common to +// both [Transport] and [Server]. +type HTTP2Config struct { + // MaxConcurrentStreams optionally specifies the number of + // concurrent streams that a peer may have open at a time. + // If zero, MaxConcurrentStreams defaults to at least 100. + MaxConcurrentStreams int + + // MaxDecoderHeaderTableSize optionally specifies an upper limit for the + // size of the header compression table used for decoding headers sent + // by the peer. + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxDecoderHeaderTableSize int + + // MaxEncoderHeaderTableSize optionally specifies an upper limit for the + // header compression table used for sending headers to the peer. + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxEncoderHeaderTableSize int + + // MaxReadFrameSize optionally specifies the largest frame + // this endpoint is willing to read. + // A valid value is between 16KiB and 16MiB, inclusive. + // If zero or invalid, a default value is used. + MaxReadFrameSize int + + // MaxReceiveBufferPerConnection is the maximum size of the + // flow control window for data received on a connection. + // A valid value is at least 64KiB and less than 4MiB. + // If invalid, a default value is used. + MaxReceiveBufferPerConnection int + + // MaxReceiveBufferPerStream is the maximum size of + // the flow control window for data received on a stream (request). + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxReceiveBufferPerStream int + + // SendPingTimeout is the timeout after which a health check using a ping + // frame will be carried out if no frame is received on a connection. + // If zero, no health check is performed. + SendPingTimeout time.Duration + + // PingTimeout is the timeout after which a connection will be closed + // if a response to a ping is not received. + // If zero, a default of 15 seconds is used. + PingTimeout time.Duration + + // WriteByteTimeout is the timeout after which a connection will be + // closed if no data can be written to it. The timeout begins when data is + // available to write, and is extended whenever any bytes are written. + WriteByteTimeout time.Duration + + // PermitProhibitedCipherSuites, if true, permits the use of + // cipher suites prohibited by the HTTP/2 spec. + PermitProhibitedCipherSuites bool + + // CountError, if non-nil, is called on HTTP/2 errors. + // It is intended to increment a metric for monitoring. + // The errType contains only lowercase letters, digits, and underscores + // (a-z, 0-9, _). + CountError func(errType string) +} diff --git a/src/net/http/http_test.go b/src/net/http/http_test.go index 2e7e024e..5aba3ed5 100644 --- a/src/net/http/http_test.go +++ b/src/net/http/http_test.go @@ -12,8 +12,8 @@ import ( "io/fs" "net/url" "os" - "reflect" "regexp" + "slices" "strings" "testing" ) @@ -41,7 +41,7 @@ func TestForeachHeaderElement(t *testing.T) { foreachHeaderElement(tt.in, func(v string) { got = append(got, v) }) - if !reflect.DeepEqual(got, tt.want) { + if !slices.Equal(got, tt.want) { t.Errorf("foreachHeaderElement(%q) = %q; want %q", tt.in, got, tt.want) } } @@ -151,9 +151,7 @@ var forbiddenStringsFunctions = map[string]bool{ // strings and bytes package functions. HTTP is mostly ASCII based, and doing // Unicode-aware case folding or space stripping can introduce vulnerabilities. func TestNoUnicodeStrings(t *testing.T) { - if !testenv.HasSrc() { - t.Skip("source code not available") - } + testenv.MustHaveSource(t) re := regexp.MustCompile(`(strings|bytes).([A-Za-z]+)`) if err := fs.WalkDir(os.DirFS("."), ".", func(path string, d fs.DirEntry, err error) error { @@ -189,6 +187,28 @@ func TestNoUnicodeStrings(t *testing.T) { } } +func TestProtocols(t *testing.T) { + var p Protocols + if p.HTTP1() { + t.Errorf("zero-value protocols: p.HTTP1() = true, want false") + } + p.SetHTTP1(true) + p.SetHTTP2(true) + if !p.HTTP1() { + t.Errorf("initialized protocols: p.HTTP1() = false, want true") + } + if !p.HTTP2() { + t.Errorf("initialized protocols: p.HTTP2() = false, want true") + } + p.SetHTTP1(false) + if p.HTTP1() { + t.Errorf("after unsetting HTTP1: p.HTTP1() = true, want false") + } + if !p.HTTP2() { + t.Errorf("after unsetting HTTP1: p.HTTP2() = false, want true") + } +} + const redirectURL = "/thisaredirect细雪withasciilettersのけぶabcdefghijk.html" func BenchmarkHexEscapeNonASCII(b *testing.B) { diff --git a/src/net/http/httputil/reverseproxy.go b/src/net/http/httputil/reverseproxy.go index 04248d5f..d64d2fc3 100644 --- a/src/net/http/httputil/reverseproxy.go +++ b/src/net/http/httputil/reverseproxy.go @@ -739,6 +739,7 @@ func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.R resUpType := upgradeType(res.Header) if !ascii.IsPrint(resUpType) { // We know reqUpType is ASCII, it's checked by the caller. p.getErrorHandler()(rw, req, fmt.Errorf("backend tried to switch to invalid protocol %q", resUpType)) + return } if !ascii.EqualFold(reqUpType, resUpType) { p.getErrorHandler()(rw, req, fmt.Errorf("backend tried to switch protocol %q when %q was requested", resUpType, reqUpType)) diff --git a/src/net/http/httputil/reverseproxy_test.go b/src/net/http/httputil/reverseproxy_test.go index eac8b7ec..2f9a5eec 100644 --- a/src/net/http/httputil/reverseproxy_test.go +++ b/src/net/http/httputil/reverseproxy_test.go @@ -137,6 +137,7 @@ func TestReverseProxy(t *testing.T) { if g, e := res.Trailer.Get("X-Unannounced-Trailer"), "unannounced_trailer_value"; g != e { t.Errorf("Trailer(X-Unannounced-Trailer) = %q ; want %q", g, e) } + res.Body.Close() // Test that a backend failing to be reached or one which doesn't return // a response results in a StatusBadGateway. @@ -205,7 +206,7 @@ func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { slices.Sort(cf) expectedValues := []string{"Upgrade", someConnHeader, fakeConnectionToken} slices.Sort(expectedValues) - if !reflect.DeepEqual(cf, expectedValues) { + if !slices.Equal(cf, expectedValues) { t.Errorf("handler modified header %q = %q; want %q", "Connection", cf, expectedValues) } })) @@ -328,6 +329,7 @@ func TestXForwardedFor(t *testing.T) { if err != nil { t.Fatalf("Get: %v", err) } + defer res.Body.Close() if g, e := res.StatusCode, backendStatus; g != e { t.Errorf("got res.StatusCode %d; expected %d", g, e) } @@ -765,7 +767,7 @@ func TestReverseProxyGetPutBuffer(t *testing.T) { wantLog := []string{"getBuf", "putBuf-" + strconv.Itoa(size)} mu.Lock() defer mu.Unlock() - if !reflect.DeepEqual(log, wantLog) { + if !slices.Equal(log, wantLog) { t.Errorf("Log events = %q; want %q", log, wantLog) } } @@ -801,6 +803,7 @@ func TestReverseProxy_Post(t *testing.T) { if err != nil { t.Fatalf("Do: %v", err) } + defer res.Body.Close() if g, e := res.StatusCode, backendStatus; g != e { t.Errorf("got res.StatusCode %d; expected %d", g, e) } @@ -1571,7 +1574,7 @@ func TestUnannouncedTrailer(t *testing.T) { } io.ReadAll(res.Body) - + res.Body.Close() if g, w := res.Trailer.Get("X-Unannounced-Trailer"), "unannounced_trailer_value"; g != w { t.Errorf("Trailer(X-Unannounced-Trailer) = %q; want %q", g, w) } diff --git a/src/net/http/netconn_test.go b/src/net/http/netconn_test.go new file mode 100644 index 00000000..ed02b98d --- /dev/null +++ b/src/net/http/netconn_test.go @@ -0,0 +1,436 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "bytes" + "context" + "internal/synctest" + "io" + "math" + "net" + "net/netip" + "os" + "sync" + "time" +) + +func fakeNetListen() *fakeNetListener { + li := &fakeNetListener{ + setc: make(chan struct{}, 1), + unsetc: make(chan struct{}, 1), + addr: netip.MustParseAddrPort("127.0.0.1:8000"), + locPort: 10000, + } + li.unsetc <- struct{}{} + return li +} + +type fakeNetListener struct { + setc, unsetc chan struct{} + queue []net.Conn + closed bool + addr netip.AddrPort + locPort uint16 + + onDial func() // called when making a new connection + + trackConns bool // set this to record all created conns + conns []*fakeNetConn +} + +func (li *fakeNetListener) lock() { + select { + case <-li.setc: + case <-li.unsetc: + } +} + +func (li *fakeNetListener) unlock() { + if li.closed || len(li.queue) > 0 { + li.setc <- struct{}{} + } else { + li.unsetc <- struct{}{} + } +} + +func (li *fakeNetListener) connect() *fakeNetConn { + if li.onDial != nil { + li.onDial() + } + li.lock() + defer li.unlock() + locAddr := netip.AddrPortFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), li.locPort) + li.locPort++ + c0, c1 := fakeNetPipe(li.addr, locAddr) + li.queue = append(li.queue, c0) + if li.trackConns { + li.conns = append(li.conns, c0) + } + return c1 +} + +func (li *fakeNetListener) Accept() (net.Conn, error) { + <-li.setc + defer li.unlock() + if li.closed { + return nil, net.ErrClosed + } + c := li.queue[0] + li.queue = li.queue[1:] + return c, nil +} + +func (li *fakeNetListener) Close() error { + li.lock() + defer li.unlock() + li.closed = true + return nil +} + +func (li *fakeNetListener) Addr() net.Addr { + return net.TCPAddrFromAddrPort(li.addr) +} + +// fakeNetPipe creates an in-memory, full duplex network connection. +// +// Unlike net.Pipe, the connection is not synchronous. +// Writes are made to a buffer, and return immediately. +// By default, the buffer size is unlimited. +func fakeNetPipe(s1ap, s2ap netip.AddrPort) (r, w *fakeNetConn) { + s1addr := net.TCPAddrFromAddrPort(s1ap) + s2addr := net.TCPAddrFromAddrPort(s2ap) + s1 := newSynctestNetConnHalf(s1addr) + s2 := newSynctestNetConnHalf(s2addr) + c1 := &fakeNetConn{loc: s1, rem: s2} + c2 := &fakeNetConn{loc: s2, rem: s1} + c1.peer = c2 + c2.peer = c1 + return c1, c2 +} + +// A fakeNetConn is one endpoint of the connection created by fakeNetPipe. +type fakeNetConn struct { + // local and remote connection halves. + // Each half contains a buffer. + // Reads pull from the local buffer, and writes push to the remote buffer. + loc, rem *fakeNetConnHalf + + // When set, synctest.Wait is automatically called before reads and after writes. + autoWait bool + + // peer is the other endpoint. + peer *fakeNetConn + + onClose func() // called when closing +} + +// Read reads data from the connection. +func (c *fakeNetConn) Read(b []byte) (n int, err error) { + if c.autoWait { + synctest.Wait() + } + return c.loc.read(b) +} + +// Peek returns the available unread read buffer, +// without consuming its contents. +func (c *fakeNetConn) Peek() []byte { + if c.autoWait { + synctest.Wait() + } + return c.loc.peek() +} + +// Write writes data to the connection. +func (c *fakeNetConn) Write(b []byte) (n int, err error) { + if c.autoWait { + defer synctest.Wait() + } + return c.rem.write(b) +} + +// IsClosed reports whether the peer has closed its end of the connection. +func (c *fakeNetConn) IsClosedByPeer() bool { + if c.autoWait { + synctest.Wait() + } + c.rem.lock() + defer c.rem.unlock() + // If the remote half of the conn is returning ErrClosed, + // the peer has closed the connection. + return c.rem.readErr == net.ErrClosed +} + +// Close closes the connection. +func (c *fakeNetConn) Close() error { + if c.onClose != nil { + c.onClose() + } + // Local half of the conn is now closed. + c.loc.lock() + c.loc.writeErr = net.ErrClosed + c.loc.readErr = net.ErrClosed + c.loc.buf.Reset() + c.loc.unlock() + // Remote half of the connection reads EOF after reading any remaining data. + c.rem.lock() + if c.rem.readErr != nil { + c.rem.readErr = io.EOF + } + c.rem.unlock() + if c.autoWait { + synctest.Wait() + } + return nil +} + +// LocalAddr returns the (fake) local network address. +func (c *fakeNetConn) LocalAddr() net.Addr { + return c.loc.addr +} + +// LocalAddr returns the (fake) remote network address. +func (c *fakeNetConn) RemoteAddr() net.Addr { + return c.rem.addr +} + +// SetDeadline sets the read and write deadlines for the connection. +func (c *fakeNetConn) SetDeadline(t time.Time) error { + c.SetReadDeadline(t) + c.SetWriteDeadline(t) + return nil +} + +// SetReadDeadline sets the read deadline for the connection. +func (c *fakeNetConn) SetReadDeadline(t time.Time) error { + c.loc.rctx.setDeadline(t) + return nil +} + +// SetWriteDeadline sets the write deadline for the connection. +func (c *fakeNetConn) SetWriteDeadline(t time.Time) error { + c.rem.wctx.setDeadline(t) + return nil +} + +// SetReadBufferSize sets the read buffer limit for the connection. +// Writes by the peer will block so long as the buffer is full. +func (c *fakeNetConn) SetReadBufferSize(size int) { + c.loc.setReadBufferSize(size) +} + +// fakeNetConnHalf is one data flow in the connection created by fakeNetPipe. +// Each half contains a buffer. Writes to the half push to the buffer, and reads pull from it. +type fakeNetConnHalf struct { + addr net.Addr + + // Read and write timeouts. + rctx, wctx deadlineContext + + // A half can be readable and/or writable. + // + // These four channels act as a lock, + // and allow waiting for readability/writability. + // When the half is unlocked, exactly one channel contains a value. + // When the half is locked, all channels are empty. + lockr chan struct{} // readable + lockw chan struct{} // writable + lockrw chan struct{} // readable and writable + lockc chan struct{} // neither readable nor writable + + bufMax int // maximum buffer size + buf bytes.Buffer + readErr error // error returned by reads + writeErr error // error returned by writes +} + +func newSynctestNetConnHalf(addr net.Addr) *fakeNetConnHalf { + h := &fakeNetConnHalf{ + addr: addr, + lockw: make(chan struct{}, 1), + lockr: make(chan struct{}, 1), + lockrw: make(chan struct{}, 1), + lockc: make(chan struct{}, 1), + bufMax: math.MaxInt, // unlimited + } + h.unlock() + return h +} + +// lock locks h. +func (h *fakeNetConnHalf) lock() { + select { + case <-h.lockw: // writable + case <-h.lockr: // readable + case <-h.lockrw: // readable and writable + case <-h.lockc: // neither readable nor writable + } +} + +// h unlocks h. +func (h *fakeNetConnHalf) unlock() { + canRead := h.readErr != nil || h.buf.Len() > 0 + canWrite := h.writeErr != nil || h.bufMax > h.buf.Len() + switch { + case canRead && canWrite: + h.lockrw <- struct{}{} // readable and writable + case canRead: + h.lockr <- struct{}{} // readable + case canWrite: + h.lockw <- struct{}{} // writable + default: + h.lockc <- struct{}{} // neither readable nor writable + } +} + +// waitAndLockForRead waits until h is readable and locks it. +func (h *fakeNetConnHalf) waitAndLockForRead() error { + // First a non-blocking select to see if we can make immediate progress. + // This permits using a canceled context for a non-blocking operation. + select { + case <-h.lockr: + return nil // readable + case <-h.lockrw: + return nil // readable and writable + default: + } + ctx := h.rctx.context() + select { + case <-h.lockr: + return nil // readable + case <-h.lockrw: + return nil // readable and writable + case <-ctx.Done(): + return context.Cause(ctx) + } +} + +// waitAndLockForWrite waits until h is writable and locks it. +func (h *fakeNetConnHalf) waitAndLockForWrite() error { + // First a non-blocking select to see if we can make immediate progress. + // This permits using a canceled context for a non-blocking operation. + select { + case <-h.lockw: + return nil // writable + case <-h.lockrw: + return nil // readable and writable + default: + } + ctx := h.wctx.context() + select { + case <-h.lockw: + return nil // writable + case <-h.lockrw: + return nil // readable and writable + case <-ctx.Done(): + return context.Cause(ctx) + } +} + +func (h *fakeNetConnHalf) peek() []byte { + h.lock() + defer h.unlock() + return h.buf.Bytes() +} + +func (h *fakeNetConnHalf) read(b []byte) (n int, err error) { + if err := h.waitAndLockForRead(); err != nil { + return 0, err + } + defer h.unlock() + if h.buf.Len() == 0 && h.readErr != nil { + return 0, h.readErr + } + return h.buf.Read(b) +} + +func (h *fakeNetConnHalf) setReadBufferSize(size int) { + h.lock() + defer h.unlock() + h.bufMax = size +} + +func (h *fakeNetConnHalf) write(b []byte) (n int, err error) { + for n < len(b) { + nn, err := h.writePartial(b[n:]) + n += nn + if err != nil { + return n, err + } + } + return n, nil +} + +func (h *fakeNetConnHalf) writePartial(b []byte) (n int, err error) { + if err := h.waitAndLockForWrite(); err != nil { + return 0, err + } + defer h.unlock() + if h.writeErr != nil { + return 0, h.writeErr + } + writeMax := h.bufMax - h.buf.Len() + if writeMax < len(b) { + b = b[:writeMax] + } + return h.buf.Write(b) +} + +// deadlineContext converts a changable deadline (as in net.Conn.SetDeadline) into a Context. +type deadlineContext struct { + mu sync.Mutex + ctx context.Context + cancel context.CancelCauseFunc + timer *time.Timer +} + +// context returns a Context which expires when the deadline does. +func (t *deadlineContext) context() context.Context { + t.mu.Lock() + defer t.mu.Unlock() + if t.ctx == nil { + t.ctx, t.cancel = context.WithCancelCause(context.Background()) + } + return t.ctx +} + +// setDeadline sets the current deadline. +func (t *deadlineContext) setDeadline(deadline time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + // If t.ctx is non-nil and t.cancel is nil, then t.ctx was canceled + // and we should create a new one. + if t.ctx == nil || t.cancel == nil { + t.ctx, t.cancel = context.WithCancelCause(context.Background()) + } + // Stop any existing deadline from expiring. + if t.timer != nil { + t.timer.Stop() + } + if deadline.IsZero() { + // No deadline. + return + } + now := time.Now() + if !deadline.After(now) { + // Deadline has already expired. + t.cancel(os.ErrDeadlineExceeded) + t.cancel = nil + return + } + if t.timer != nil { + // Reuse existing deadline timer. + t.timer.Reset(deadline.Sub(now)) + return + } + // Create a new timer to cancel the context at the deadline. + t.timer = time.AfterFunc(deadline.Sub(now), func() { + t.mu.Lock() + defer t.mu.Unlock() + t.cancel(os.ErrDeadlineExceeded) + t.cancel = nil + }) +} diff --git a/src/net/http/pprof/pprof.go b/src/net/http/pprof/pprof.go index cf4b8415..6ba6b2c8 100644 --- a/src/net/http/pprof/pprof.go +++ b/src/net/http/pprof/pprof.go @@ -86,7 +86,7 @@ import ( "runtime" "runtime/pprof" "runtime/trace" - "sort" + "slices" "strconv" "strings" "time" @@ -367,6 +367,7 @@ var profileDescriptions = map[string]string{ "heap": "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.", "mutex": "Stack traces of holders of contended mutexes", "profile": "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.", + "symbol": "Maps given program counters to function names. Counters can be specified in a GET raw query or POST body, multiple counters are separated by '+'.", "threadcreate": "Stack traces that led to the creation of new OS threads", "trace": "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.", } @@ -404,7 +405,7 @@ func Index(w http.ResponseWriter, r *http.Request) { } // Adding other profiles exposed from within this package - for _, p := range []string{"cmdline", "profile", "trace"} { + for _, p := range []string{"cmdline", "profile", "symbol", "trace"} { profiles = append(profiles, profileEntry{ Name: p, Href: p, @@ -412,8 +413,8 @@ func Index(w http.ResponseWriter, r *http.Request) { }) } - sort.Slice(profiles, func(i, j int) bool { - return profiles[i].Name < profiles[j].Name + slices.SortFunc(profiles, func(a, b profileEntry) int { + return strings.Compare(a.Name, b.Name) }) if err := indexTmplExecute(w, profiles); err != nil { diff --git a/src/net/http/request.go b/src/net/http/request.go index ad1b5a62..434c1640 100644 --- a/src/net/http/request.go +++ b/src/net/http/request.go @@ -15,6 +15,7 @@ import ( "errors" "fmt" "io" + "maps" "mime" "mime/multipart" "net/http/httptrace" @@ -390,12 +391,8 @@ func (r *Request) Clone(ctx context.Context) *Request { *r2 = *r r2.ctx = ctx r2.URL = cloneURL(r.URL) - if r.Header != nil { - r2.Header = r.Header.Clone() - } - if r.Trailer != nil { - r2.Trailer = r.Trailer.Clone() - } + r2.Header = r.Header.Clone() + r2.Trailer = r.Trailer.Clone() if s := r.TransferEncoding; s != nil { s2 := make([]string, len(s)) copy(s2, s) @@ -411,13 +408,7 @@ func (r *Request) Clone(ctx context.Context) *Request { copy(s2, s) r2.matches = s2 } - if s := r.otherValues; s != nil { - s2 := make(map[string]string, len(s)) - for k, v := range s { - s2[k] = v - } - r2.otherValues = s2 - } + r2.otherValues = maps.Clone(r.otherValues) return r2 } @@ -882,9 +873,9 @@ func NewRequest(method, url string, body io.Reader) (*Request, error) { // // NewRequestWithContext returns a Request suitable for use with // [Client.Do] or [Transport.RoundTrip]. To create a request for use with -// testing a Server Handler, either use the [NewRequest] function in the -// net/http/httptest package, use [ReadRequest], or manually update the -// Request fields. For an outgoing client request, the context +// testing a Server Handler, either use the [net/http/httptest.NewRequest] function, +// use [ReadRequest], or manually update the Request fields. +// For an outgoing client request, the context // controls the entire lifetime of a request and its response: // obtaining a connection, sending the request, and reading the // response headers and body. See the Request type's documentation for diff --git a/src/net/http/request_test.go b/src/net/http/request_test.go index 9b6eb6e1..37b88831 100644 --- a/src/net/http/request_test.go +++ b/src/net/http/request_test.go @@ -23,6 +23,7 @@ import ( "os" "reflect" "regexp" + "slices" "strings" "testing" ) @@ -69,22 +70,22 @@ func TestParseFormQuery(t *testing.T) { if bz := req.PostFormValue("z"); bz != "post" { t.Errorf(`req.PostFormValue("z") = %q, want "post"`, bz) } - if qs := req.Form["q"]; !reflect.DeepEqual(qs, []string{"foo", "bar"}) { + if qs := req.Form["q"]; !slices.Equal(qs, []string{"foo", "bar"}) { t.Errorf(`req.Form["q"] = %q, want ["foo", "bar"]`, qs) } - if both := req.Form["both"]; !reflect.DeepEqual(both, []string{"y", "x"}) { + if both := req.Form["both"]; !slices.Equal(both, []string{"y", "x"}) { t.Errorf(`req.Form["both"] = %q, want ["y", "x"]`, both) } if prio := req.FormValue("prio"); prio != "2" { t.Errorf(`req.FormValue("prio") = %q, want "2" (from body)`, prio) } - if orphan := req.Form["orphan"]; !reflect.DeepEqual(orphan, []string{"", "nope"}) { + if orphan := req.Form["orphan"]; !slices.Equal(orphan, []string{"", "nope"}) { t.Errorf(`req.FormValue("orphan") = %q, want "" (from body)`, orphan) } - if empty := req.Form["empty"]; !reflect.DeepEqual(empty, []string{"", "not"}) { + if empty := req.Form["empty"]; !slices.Equal(empty, []string{"", "not"}) { t.Errorf(`req.FormValue("empty") = %q, want "" (from body)`, empty) } - if nokey := req.Form[""]; !reflect.DeepEqual(nokey, []string{"nokey"}) { + if nokey := req.Form[""]; !slices.Equal(nokey, []string{"nokey"}) { t.Errorf(`req.FormValue("nokey") = %q, want "nokey" (from body)`, nokey) } } @@ -765,7 +766,7 @@ func TestRequestWriteBufferedWriter(t *testing.T) { "User-Agent: " + DefaultUserAgent + "\r\n", "\r\n", } - if !reflect.DeepEqual(got, want) { + if !slices.Equal(got, want) { t.Errorf("Writes = %q\n Want = %q", got, want) } } @@ -785,7 +786,7 @@ func TestRequestBadHostHeader(t *testing.T) { "User-Agent: " + DefaultUserAgent + "\r\n", "\r\n", } - if !reflect.DeepEqual(got, want) { + if !slices.Equal(got, want) { t.Errorf("Writes = %q\n Want = %q", got, want) } } @@ -804,7 +805,7 @@ func TestRequestBadUserAgent(t *testing.T) { "User-Agent: evil X-Evil: evil\r\n", "\r\n", } - if !reflect.DeepEqual(got, want) { + if !slices.Equal(got, want) { t.Errorf("Writes = %q\n Want = %q", got, want) } } diff --git a/src/net/http/response_test.go b/src/net/http/response_test.go index a63aac95..3ccbb9b0 100644 --- a/src/net/http/response_test.go +++ b/src/net/http/response_test.go @@ -21,9 +21,10 @@ import ( ) type respTest struct { - Raw string - Resp Response - Body string + Raw string + RawOut string + Resp Response + Body string } func dummyReq(method string) *Request { @@ -42,6 +43,11 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -66,6 +72,11 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -87,6 +98,9 @@ var respTests = []respTest{ "\r\n" + "Body should not be read!\n", + "HTTP/1.1 204 No Content\r\n" + + "\r\n", + Response{ Status: "204 No Content", StatusCode: 204, @@ -110,6 +124,12 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Content-Length: 10\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -140,6 +160,14 @@ var respTests = []respTest{ "0\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n" + + "13\r\n" + + "Body here\ncontinued\r\n" + + "0\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -165,6 +193,12 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Content-Length: 10\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -195,6 +229,14 @@ var respTests = []respTest{ "0\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n" + + "a\r\n" + + "Body here\n\r\n" + + "0\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -217,6 +259,10 @@ var respTests = []respTest{ "Transfer-Encoding: chunked\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -239,6 +285,11 @@ var respTests = []respTest{ "Content-Length: 256\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -261,6 +312,10 @@ var respTests = []respTest{ "Content-Length: 256\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -282,6 +337,10 @@ var respTests = []respTest{ "HTTP/1.0 200 OK\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -304,6 +363,10 @@ var respTests = []respTest{ "Content-Length: 0\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 0\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -325,6 +388,11 @@ var respTests = []respTest{ // (permitted by RFC 7230, section 3.1.2) { "HTTP/1.0 303 \r\n\r\n", + + "HTTP/1.0 303 \r\n" + + "Connection: close\r\n" + + "\r\n", + Response{ Status: "303 ", StatusCode: 303, @@ -344,6 +412,11 @@ var respTests = []respTest{ // (not permitted by RFC 7230, but we'll accept it anyway) { "HTTP/1.0 303\r\n\r\n", + + "HTTP/1.0 303 303\r\n" + + "Connection: close\r\n" + + "\r\n", + Response{ Status: "303", StatusCode: 303, @@ -366,6 +439,13 @@ Connection: close Content-Type: multipart/byteranges; boundary=18a75608c8f47cef some body`, + + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Content-Type: multipart/byteranges; boundary=18a75608c8f47cef\r\n" + + "\r\n" + + "some body", + Response{ Status: "206 Partial Content", StatusCode: 206, @@ -390,6 +470,11 @@ some body`, "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -415,6 +500,14 @@ some body`, "Content-Length: 6\r\n\r\n" + "foobar", + "HTTP/1.1 206 Partial Content\r\n" + + "Content-Length: 6\r\n" + + "Accept-Ranges: bytes\r\n" + + "Content-Range: bytes 0-5/1862\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + + "\r\n" + + "foobar", + Response{ Status: "206 Partial Content", StatusCode: 206, @@ -441,6 +534,11 @@ some body`, "Connection: keep-alive, close\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -467,6 +565,11 @@ some body`, "Connection: close\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -493,6 +596,11 @@ some body`, "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -517,6 +625,12 @@ some body`, "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 10\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -541,6 +655,14 @@ some body`, "Connection: keep-alive\r\n" + "Keep-Alive: timeout=7200\r\n\r\n" + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 23\r\n" + + "Connection: keep-alive\r\n" + + "Content-Encoding: gzip\r\n" + + "Keep-Alive: timeout=7200\r\n\r\n" + + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + Response{ Status: "200 OK", StatusCode: 200, @@ -566,6 +688,14 @@ some body`, "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"\"\r\n\r\n" + "Your Authentication failed.\r\n", + + "HTTP/1.0 401 Unauthorized\r\n" + + "Connection: close\r\n" + + "Content-Type: text/html\r\n" + + "Www-Authenticate: Basic realm=\"\"\r\n" + + "\r\n" + + "Your Authentication failed.\r\n", + Response{ Status: "401 Unauthorized", StatusCode: 401, @@ -619,11 +749,18 @@ func TestWriteResponse(t *testing.T) { t.Errorf("#%d: %v", i, err) continue } - err = resp.Write(io.Discard) + var buf bytes.Buffer + err = resp.Write(&buf) if err != nil { t.Errorf("#%d: %v", i, err) continue } + if got, want := buf.String(), tt.RawOut; got != want { + t.Errorf("#%d: response differs; got:\n----\n%v\n----\nwant:\n----\n%v\n----\n", + i, + strings.ReplaceAll(got, "\r", "\\r"), + strings.ReplaceAll(want, "\r", "\\r")) + } } } diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index b2858ba8..0c46b1ec 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -16,6 +16,7 @@ import ( "encoding/json" "errors" "fmt" + "internal/synctest" "internal/testenv" "io" "log" @@ -34,6 +35,7 @@ import ( "reflect" "regexp" "runtime" + "slices" "strconv" "strings" "sync" @@ -1509,7 +1511,7 @@ func testHeadResponses(t *testing.T, mode testMode) { } // Also exercise the ReaderFrom path - _, err = io.Copy(w, strings.NewReader("789a")) + _, err = io.Copy(w, struct{ io.Reader }{strings.NewReader("789a")}) if err != nil { t.Errorf("Copy(ResponseWriter, ...): %v", err) } @@ -1536,6 +1538,34 @@ func testHeadResponses(t *testing.T, mode testMode) { } } +// Ensure ResponseWriter.ReadFrom doesn't write a body in response to a HEAD request. +// https://go.dev/issue/68609 +func TestHeadReaderFrom(t *testing.T) { run(t, testHeadReaderFrom, []testMode{http1Mode}) } +func testHeadReaderFrom(t *testing.T, mode testMode) { + // Body is large enough to exceed the content-sniffing length. + wantBody := strings.Repeat("a", 4096) + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.(io.ReaderFrom).ReadFrom(strings.NewReader(wantBody)) + })) + res, err := cst.c.Head(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + res, err = cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + gotBody, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatal(err) + } + if string(gotBody) != wantBody { + t.Errorf("got unexpected body len=%v, want %v", len(gotBody), len(wantBody)) + } +} + func TestTLSHandshakeTimeout(t *testing.T) { run(t, testTLSHandshakeTimeout, []testMode{https1Mode, http2Mode}) } @@ -4010,7 +4040,7 @@ func testHTTP10ConnectionHeader(t *testing.T, mode testMode) { resp.Body.Close() got := resp.Header["Connection"] - if !reflect.DeepEqual(got, tt.expect) { + if !slices.Equal(got, tt.expect) { t.Errorf("wrong Connection headers for request %q. Got %q expect %q", tt.req, got, tt.expect) } } @@ -4329,7 +4359,7 @@ func testServerConnState(t *testing.T, mode testMode) { <-complete sl := <-activeLog - if !reflect.DeepEqual(sl.got, sl.want) { + if !slices.Equal(sl.got, sl.want) { t.Errorf("Request(s) produced unexpected state sequence.\nGot: %v\nWant: %v", sl.got, sl.want) } // Don't return sl to activeLog: we don't expect any further states after @@ -4355,7 +4385,7 @@ func testServerConnState(t *testing.T, mode testMode) { return } sl.got = append(sl.got, state) - if sl.complete != nil && (len(sl.got) >= len(sl.want) || !reflect.DeepEqual(sl.got, sl.want[:len(sl.got)])) { + if sl.complete != nil && (len(sl.got) >= len(sl.want) || !slices.Equal(sl.got, sl.want[:len(sl.got)])) { close(sl.complete) sl.complete = nil } @@ -5776,70 +5806,59 @@ func testServerShutdown(t *testing.T, mode testMode) { } } -func TestServerShutdownStateNew(t *testing.T) { run(t, testServerShutdownStateNew) } -func testServerShutdownStateNew(t *testing.T, mode testMode) { +func TestServerShutdownStateNew(t *testing.T) { runSynctest(t, testServerShutdownStateNew) } +func testServerShutdownStateNew(t testing.TB, mode testMode) { if testing.Short() { t.Skip("test takes 5-6 seconds; skipping in short mode") } - var connAccepted sync.WaitGroup + listener := fakeNetListen() + defer listener.Close() + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { // nothing. }), func(ts *httptest.Server) { - ts.Config.ConnState = func(conn net.Conn, state ConnState) { - if state == StateNew { - connAccepted.Done() - } - } + ts.Listener.Close() + ts.Listener = listener + // Ignore irrelevant error about TLS handshake failure. + ts.Config.ErrorLog = log.New(io.Discard, "", 0) }).ts // Start a connection but never write to it. - connAccepted.Add(1) - c, err := net.Dial("tcp", ts.Listener.Addr().String()) - if err != nil { - t.Fatal(err) - } + c := listener.connect() defer c.Close() + synctest.Wait() - // Wait for the connection to be accepted by the server. Otherwise, if - // Shutdown happens to run first, the server will be closed when - // encountering the connection, in which case it will be rejected - // immediately. - connAccepted.Wait() - - shutdownRes := make(chan error, 1) - go func() { - shutdownRes <- ts.Config.Shutdown(context.Background()) - }() - readRes := make(chan error, 1) - go func() { - _, err := c.Read([]byte{0}) - readRes <- err - }() + shutdownRes := runAsync(func() (struct{}, error) { + return struct{}{}, ts.Config.Shutdown(context.Background()) + }) // TODO(#59037): This timeout is hard-coded in closeIdleConnections. // It is undocumented, and some users may find it surprising. // Either document it, or switch to a less surprising behavior. const expectTimeout = 5 * time.Second - t0 := time.Now() - select { - case got := <-shutdownRes: - d := time.Since(t0) - if got != nil { - t.Fatalf("shutdown error after %v: %v", d, err) - } - if d < expectTimeout/2 { - t.Errorf("shutdown too soon after %v", d) - } - case <-time.After(expectTimeout * 3 / 2): - t.Fatalf("timeout waiting for shutdown") + // Wait until just before the expected timeout. + time.Sleep(expectTimeout - 1) + synctest.Wait() + if shutdownRes.done() { + t.Fatal("shutdown too soon") + } + if c.IsClosedByPeer() { + t.Fatal("connection was closed by server too soon") } - // Wait for c.Read to unblock; should be already done at this point, - // or within a few milliseconds. - if err := <-readRes; err == nil { - t.Error("expected error from Read") + // closeIdleConnections isn't precise about its actual shutdown time. + // Wait long enough for it to definitely have shut down. + // + // (It would be good to make closeIdleConnections less sloppy.) + time.Sleep(2 * time.Second) + synctest.Wait() + if _, err := shutdownRes.result(); err != nil { + t.Fatalf("Shutdown() = %v, want complete", err) + } + if !c.IsClosedByPeer() { + t.Fatalf("connection was not closed by server after shutdown") } } diff --git a/src/net/http/server.go b/src/net/http/server.go index 1ff72a04..1e8e1437 100644 --- a/src/net/http/server.go +++ b/src/net/http/server.go @@ -611,7 +611,7 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { w.cw.flush() // make sure Header is written; flush data to rwc // Now that cw has been flushed, its chunking field is guaranteed initialized. - if !w.cw.chunking && w.bodyAllowed() { + if !w.cw.chunking && w.bodyAllowed() && w.req.Method != "HEAD" { n0, err := rf.ReadFrom(src) n += n0 w.written += n0 @@ -628,9 +628,9 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { const debugServerConnections = false // Create new connection from rwc. -func (srv *Server) newConn(rwc net.Conn) *conn { +func (s *Server) newConn(rwc net.Conn) *conn { c := &conn{ - server: srv, + server: s, rwc: rwc, } if debugServerConnections { @@ -915,15 +915,15 @@ func putBufioWriter(bw *bufio.Writer) { // This can be overridden by setting [Server.MaxHeaderBytes]. const DefaultMaxHeaderBytes = 1 << 20 // 1 MB -func (srv *Server) maxHeaderBytes() int { - if srv.MaxHeaderBytes > 0 { - return srv.MaxHeaderBytes +func (s *Server) maxHeaderBytes() int { + if s.MaxHeaderBytes > 0 { + return s.MaxHeaderBytes } return DefaultMaxHeaderBytes } -func (srv *Server) initialReadLimitSize() int64 { - return int64(srv.maxHeaderBytes()) + 4096 // bufio slop +func (s *Server) initialReadLimitSize() int64 { + return int64(s.maxHeaderBytes()) + 4096 // bufio slop } // tlsHandshakeTimeout returns the time limit permitted for the TLS @@ -931,12 +931,12 @@ func (srv *Server) initialReadLimitSize() int64 { // // It returns the minimum of any positive ReadHeaderTimeout, // ReadTimeout, or WriteTimeout. -func (srv *Server) tlsHandshakeTimeout() time.Duration { +func (s *Server) tlsHandshakeTimeout() time.Duration { var ret time.Duration for _, v := range [...]time.Duration{ - srv.ReadHeaderTimeout, - srv.ReadTimeout, - srv.WriteTimeout, + s.ReadHeaderTimeout, + s.ReadTimeout, + s.WriteTimeout, } { if v <= 0 { continue @@ -2013,6 +2013,16 @@ func (c *conn) serve(ctx context.Context) { c.bufr = newBufioReader(c.r) c.bufw = newBufioWriterSize(checkConnErrorWriter{c}, 4<<10) + protos := c.server.protocols() + if c.tlsState == nil && protos.UnencryptedHTTP2() { + if c.maybeServeUnencryptedHTTP2(ctx) { + return + } + } + if !protos.HTTP1() { + return + } + for { w, err := c.readRequest(ctx) if c.r.remain != c.server.initialReadLimitSize() { @@ -2132,6 +2142,70 @@ func (c *conn) serve(ctx context.Context) { } } +// unencryptedHTTP2Request is an HTTP handler that initializes +// certain uninitialized fields in its *Request. +// +// It's the unencrypted version of initALPNRequest. +type unencryptedHTTP2Request struct { + ctx context.Context + c net.Conn + h serverHandler +} + +func (h unencryptedHTTP2Request) BaseContext() context.Context { return h.ctx } + +func (h unencryptedHTTP2Request) ServeHTTP(rw ResponseWriter, req *Request) { + if req.Body == nil { + req.Body = NoBody + } + if req.RemoteAddr == "" { + req.RemoteAddr = h.c.RemoteAddr().String() + } + h.h.ServeHTTP(rw, req) +} + +// unencryptedNetConnInTLSConn is used to pass an unencrypted net.Conn to +// functions that only accept a *tls.Conn. +type unencryptedNetConnInTLSConn struct { + net.Conn // panic on all net.Conn methods + conn net.Conn +} + +func (c unencryptedNetConnInTLSConn) UnencryptedNetConn() net.Conn { + return c.conn +} + +func unencryptedTLSConn(c net.Conn) *tls.Conn { + return tls.Client(unencryptedNetConnInTLSConn{conn: c}, nil) +} + +// TLSNextProto key to use for unencrypted HTTP/2 connections. +// Not actually a TLS-negotiated protocol. +const nextProtoUnencryptedHTTP2 = "unencrypted_http2" + +func (c *conn) maybeServeUnencryptedHTTP2(ctx context.Context) bool { + fn, ok := c.server.TLSNextProto[nextProtoUnencryptedHTTP2] + if !ok { + return false + } + hasPreface := func(c *conn, preface []byte) bool { + c.r.setReadLimit(int64(len(preface)) - int64(c.bufr.Buffered())) + got, err := c.bufr.Peek(len(preface)) + c.r.setInfiniteReadLimit() + return err == nil && bytes.Equal(got, preface) + } + if !hasPreface(c, []byte("PRI * HTTP/2.0")) { + return false + } + if !hasPreface(c, []byte("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n")) { + return false + } + c.setState(c.rwc, StateActive, skipHooks) + h := unencryptedHTTP2Request{ctx, c.rwc, serverHandler{c.server}} + fn(c.server, unencryptedTLSConn(c.rwc), h) + return true +} + func (w *response) sendExpectationFailed() { // TODO(bradfitz): let ServeHTTP handlers handle // requests with non-standard expectation[s]? Seems @@ -2480,6 +2554,8 @@ func RedirectHandler(url string, code int) Handler { // ServeMux also takes care of sanitizing the URL request path and the Host // header, stripping the port number and redirecting any request containing . or // .. segments or repeated slashes to an equivalent, cleaner URL. +// Escaped path elements such as "%2e" for "." and "%2f" for "/" are preserved +// and aren't considered separators for request routing. // // # Compatibility // @@ -2503,11 +2579,10 @@ func RedirectHandler(url string, code int) Handler { // This change mostly affects how paths with %2F escapes adjacent to slashes are treated. // See https://go.dev/issue/21955 for details. type ServeMux struct { - mu sync.RWMutex - tree routingNode - index routingIndex - patterns []*pattern // TODO(jba): remove if possible - mux121 serveMux121 // used only when GODEBUG=httpmuxgo121=1 + mu sync.RWMutex + tree routingNode + index routingIndex + mux121 serveMux121 // used only when GODEBUG=httpmuxgo121=1 } // NewServeMux allocates and returns a new [ServeMux]. @@ -2838,7 +2913,6 @@ func (mux *ServeMux) registerErr(patstr string, handler Handler) error { } mux.tree.addPattern(pat, handler) mux.index.addPattern(pat) - mux.patterns = append(mux.patterns, pat) return nil } @@ -2973,6 +3047,23 @@ type Server struct { // value. ConnContext func(ctx context.Context, c net.Conn) context.Context + // HTTP2 configures HTTP/2 connections. + // + // This field does not yet have any effect. + // See https://go.dev/issue/67813. + HTTP2 *HTTP2Config + + // Protocols is the set of protocols accepted by the server. + // + // If Protocols includes UnencryptedHTTP2, the server will accept + // unencrypted HTTP/2 connections. The server can serve both + // HTTP/1 and unencrypted HTTP/2 on the same address and port. + // + // If Protocols is nil, the default is usually HTTP/1 and HTTP/2. + // If TLSNextProto is non-nil and does not contain an "h2" entry, + // the default is HTTP/1 only. + Protocols *Protocols + inShutdown atomic.Bool // true when server is in shutdown disableKeepAlives atomic.Bool @@ -2996,23 +3087,23 @@ type Server struct { // // Close returns any error returned from closing the [Server]'s // underlying Listener(s). -func (srv *Server) Close() error { - srv.inShutdown.Store(true) - srv.mu.Lock() - defer srv.mu.Unlock() - err := srv.closeListenersLocked() +func (s *Server) Close() error { + s.inShutdown.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + err := s.closeListenersLocked() - // Unlock srv.mu while waiting for listenerGroup. - // The group Add and Done calls are made with srv.mu held, + // Unlock s.mu while waiting for listenerGroup. + // The group Add and Done calls are made with s.mu held, // to avoid adding a new listener in the window between // us setting inShutdown above and waiting here. - srv.mu.Unlock() - srv.listenerGroup.Wait() - srv.mu.Lock() + s.mu.Unlock() + s.listenerGroup.Wait() + s.mu.Lock() - for c := range srv.activeConn { + for c := range s.activeConn { c.rwc.Close() - delete(srv.activeConn, c) + delete(s.activeConn, c) } return err } @@ -3046,16 +3137,16 @@ const shutdownPollIntervalMax = 500 * time.Millisecond // // Once Shutdown has been called on a server, it may not be reused; // future calls to methods such as Serve will return ErrServerClosed. -func (srv *Server) Shutdown(ctx context.Context) error { - srv.inShutdown.Store(true) +func (s *Server) Shutdown(ctx context.Context) error { + s.inShutdown.Store(true) - srv.mu.Lock() - lnerr := srv.closeListenersLocked() - for _, f := range srv.onShutdown { + s.mu.Lock() + lnerr := s.closeListenersLocked() + for _, f := range s.onShutdown { go f() } - srv.mu.Unlock() - srv.listenerGroup.Wait() + s.mu.Unlock() + s.listenerGroup.Wait() pollIntervalBase := time.Millisecond nextPollInterval := func() time.Duration { @@ -3072,7 +3163,7 @@ func (srv *Server) Shutdown(ctx context.Context) error { timer := time.NewTimer(nextPollInterval()) defer timer.Stop() for { - if srv.closeIdleConns() { + if s.closeIdleConns() { return lnerr } select { @@ -3089,10 +3180,10 @@ func (srv *Server) Shutdown(ctx context.Context) error { // undergone ALPN protocol upgrade or that have been hijacked. // This function should start protocol-specific graceful shutdown, // but should not wait for shutdown to complete. -func (srv *Server) RegisterOnShutdown(f func()) { - srv.mu.Lock() - srv.onShutdown = append(srv.onShutdown, f) - srv.mu.Unlock() +func (s *Server) RegisterOnShutdown(f func()) { + s.mu.Lock() + s.onShutdown = append(s.onShutdown, f) + s.mu.Unlock() } // closeIdleConns closes all idle connections and reports whether the @@ -3236,19 +3327,19 @@ func AllowQuerySemicolons(h Handler) Handler { }) } -// ListenAndServe listens on the TCP network address srv.Addr and then +// ListenAndServe listens on the TCP network address s.Addr and then // calls [Serve] to handle requests on incoming connections. // Accepted connections are configured to enable TCP keep-alives. // -// If srv.Addr is blank, ":http" is used. +// If s.Addr is blank, ":http" is used. // // ListenAndServe always returns a non-nil error. After [Server.Shutdown] or [Server.Close], // the returned error is [ErrServerClosed]. -func (srv *Server) ListenAndServe() error { - if srv.shuttingDown() { +func (s *Server) ListenAndServe() error { + if s.shuttingDown() { return ErrServerClosed } - addr := srv.Addr + addr := s.Addr if addr == "" { addr = ":http" } @@ -3256,23 +3347,26 @@ func (srv *Server) ListenAndServe() error { if err != nil { return err } - return srv.Serve(ln) + return s.Serve(ln) } var testHookServerServe func(*Server, net.Listener) // used if non-nil // shouldConfigureHTTP2ForServe reports whether Server.Serve should configure -// automatic HTTP/2. (which sets up the srv.TLSNextProto map) -func (srv *Server) shouldConfigureHTTP2ForServe() bool { - if srv.TLSConfig == nil { +// automatic HTTP/2. (which sets up the s.TLSNextProto map) +func (s *Server) shouldConfigureHTTP2ForServe() bool { + if s.TLSConfig == nil { // Compatibility with Go 1.6: // If there's no TLSConfig, it's possible that the user just // didn't set it on the http.Server, but did pass it to // tls.NewListener and passed that listener to Serve. - // So we should configure HTTP/2 (to set up srv.TLSNextProto) + // So we should configure HTTP/2 (to set up s.TLSNextProto) // in case the listener returns an "h2" *tls.Conn. return true } + if s.protocols().UnencryptedHTTP2() { + return true + } // The user specified a TLSConfig on their http.Server. // In this, case, only configure HTTP/2 if their tls.Config // explicitly mentions "h2". Otherwise http2.ConfigureServer @@ -3280,7 +3374,7 @@ func (srv *Server) shouldConfigureHTTP2ForServe() bool { // passed this tls.Config to tls.NewListener. And if they did, // it's too late anyway to fix it. It would only be potentially racy. // See Issue 15908. - return slices.Contains(srv.TLSConfig.NextProtos, http2NextProtoTLS) + return slices.Contains(s.TLSConfig.NextProtos, http2NextProtoTLS) } // ErrServerClosed is returned by the [Server.Serve], [ServeTLS], [ListenAndServe], @@ -3289,7 +3383,7 @@ var ErrServerClosed = errors.New("http: Server closed") // Serve accepts incoming connections on the Listener l, creating a // new service goroutine for each. The service goroutines read requests and -// then call srv.Handler to reply to them. +// then call s.Handler to reply to them. // // HTTP/2 support is only enabled if the Listener returns [*tls.Conn] // connections and they were configured with "h2" in the TLS @@ -3297,27 +3391,27 @@ var ErrServerClosed = errors.New("http: Server closed") // // Serve always returns a non-nil error and closes l. // After [Server.Shutdown] or [Server.Close], the returned error is [ErrServerClosed]. -func (srv *Server) Serve(l net.Listener) error { +func (s *Server) Serve(l net.Listener) error { if fn := testHookServerServe; fn != nil { - fn(srv, l) // call hook with unwrapped listener + fn(s, l) // call hook with unwrapped listener } origListener := l l = &onceCloseListener{Listener: l} defer l.Close() - if err := srv.setupHTTP2_Serve(); err != nil { + if err := s.setupHTTP2_Serve(); err != nil { return err } - if !srv.trackListener(&l, true) { + if !s.trackListener(&l, true) { return ErrServerClosed } - defer srv.trackListener(&l, false) + defer s.trackListener(&l, false) baseCtx := context.Background() - if srv.BaseContext != nil { - baseCtx = srv.BaseContext(origListener) + if s.BaseContext != nil { + baseCtx = s.BaseContext(origListener) if baseCtx == nil { panic("BaseContext returned a nil context") } @@ -3325,11 +3419,11 @@ func (srv *Server) Serve(l net.Listener) error { var tempDelay time.Duration // how long to sleep on accept failure - ctx := context.WithValue(baseCtx, ServerContextKey, srv) + ctx := context.WithValue(baseCtx, ServerContextKey, s) for { rw, err := l.Accept() if err != nil { - if srv.shuttingDown() { + if s.shuttingDown() { return ErrServerClosed } if ne, ok := err.(net.Error); ok && ne.Temporary() { @@ -3341,21 +3435,21 @@ func (srv *Server) Serve(l net.Listener) error { if max := 1 * time.Second; tempDelay > max { tempDelay = max } - srv.logf("http: Accept error: %v; retrying in %v", err, tempDelay) + s.logf("http: Accept error: %v; retrying in %v", err, tempDelay) time.Sleep(tempDelay) continue } return err } connCtx := ctx - if cc := srv.ConnContext; cc != nil { + if cc := s.ConnContext; cc != nil { connCtx = cc(connCtx, rw) if connCtx == nil { panic("ConnContext returned nil") } } tempDelay = 0 - c := srv.newConn(rw) + c := s.newConn(rw) c.setState(c.rwc, StateNew, runHooks) // before Serve can return go c.serve(connCtx) } @@ -3363,7 +3457,7 @@ func (srv *Server) Serve(l net.Listener) error { // ServeTLS accepts incoming connections on the Listener l, creating a // new service goroutine for each. The service goroutines perform TLS -// setup and then read requests, calling srv.Handler to reply to them. +// setup and then read requests, calling s.Handler to reply to them. // // Files containing a certificate and matching private key for the // server must be provided if neither the [Server]'s @@ -3375,17 +3469,15 @@ func (srv *Server) Serve(l net.Listener) error { // // ServeTLS always returns a non-nil error. After [Server.Shutdown] or [Server.Close], the // returned error is [ErrServerClosed]. -func (srv *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { - // Setup HTTP/2 before srv.Serve, to initialize srv.TLSConfig +func (s *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { + // Setup HTTP/2 before s.Serve, to initialize s.TLSConfig // before we clone it and create the TLS Listener. - if err := srv.setupHTTP2_ServeTLS(); err != nil { + if err := s.setupHTTP2_ServeTLS(); err != nil { return err } - config := cloneTLSConfig(srv.TLSConfig) - if !slices.Contains(config.NextProtos, "http/1.1") { - config.NextProtos = append(config.NextProtos, "http/1.1") - } + config := cloneTLSConfig(s.TLSConfig) + config.NextProtos = adjustNextProtos(config.NextProtos, s.protocols()) configHasCert := len(config.Certificates) > 0 || config.GetCertificate != nil || config.GetConfigForClient != nil if !configHasCert || certFile != "" || keyFile != "" { @@ -3398,7 +3490,60 @@ func (srv *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { } tlsListener := tls.NewListener(l, config) - return srv.Serve(tlsListener) + return s.Serve(tlsListener) +} + +func (s *Server) protocols() Protocols { + if s.Protocols != nil { + return *s.Protocols // user-configured set + } + + // The historic way of disabling HTTP/2 is to set TLSNextProto to + // a non-nil map with no "h2" entry. + _, hasH2 := s.TLSNextProto["h2"] + http2Disabled := s.TLSNextProto != nil && !hasH2 + + // If GODEBUG=http2server=0, then HTTP/2 is disabled unless + // the user has manually added an "h2" entry to TLSNextProto + // (probably by using x/net/http2 directly). + if http2server.Value() == "0" && !hasH2 { + http2Disabled = true + } + + var p Protocols + p.SetHTTP1(true) // default always includes HTTP/1 + if !http2Disabled { + p.SetHTTP2(true) + } + return p +} + +// adjustNextProtos adds or removes "http/1.1" and "h2" entries from +// a tls.Config.NextProtos list, according to the set of protocols in protos. +func adjustNextProtos(nextProtos []string, protos Protocols) []string { + var have Protocols + nextProtos = slices.DeleteFunc(nextProtos, func(s string) bool { + switch s { + case "http/1.1": + if !protos.HTTP1() { + return true + } + have.SetHTTP1(true) + case "h2": + if !protos.HTTP2() { + return true + } + have.SetHTTP2(true) + } + return false + }) + if protos.HTTP2() && !have.HTTP2() { + nextProtos = append(nextProtos, "h2") + } + if protos.HTTP1() && !have.HTTP1() { + nextProtos = append(nextProtos, "http/1.1") + } + return nextProtos } // trackListener adds or removes a net.Listener to the set of tracked @@ -3469,15 +3614,15 @@ func (s *Server) shuttingDown() bool { // By default, keep-alives are always enabled. Only very // resource-constrained environments or servers in the process of // shutting down should disable them. -func (srv *Server) SetKeepAlivesEnabled(v bool) { +func (s *Server) SetKeepAlivesEnabled(v bool) { if v { - srv.disableKeepAlives.Store(false) + s.disableKeepAlives.Store(false) return } - srv.disableKeepAlives.Store(true) + s.disableKeepAlives.Store(true) // Close idle HTTP/1 conns: - srv.closeIdleConns() + s.closeIdleConns() // TODO: Issue 26303: close HTTP/2 conns as soon as they become idle. } @@ -3524,7 +3669,7 @@ func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { return server.ListenAndServeTLS(certFile, keyFile) } -// ListenAndServeTLS listens on the TCP network address srv.Addr and +// ListenAndServeTLS listens on the TCP network address s.Addr and // then calls [ServeTLS] to handle requests on incoming TLS connections. // Accepted connections are configured to enable TCP keep-alives. // @@ -3535,15 +3680,15 @@ func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { // concatenation of the server's certificate, any intermediates, and // the CA's certificate. // -// If srv.Addr is blank, ":https" is used. +// If s.Addr is blank, ":https" is used. // // ListenAndServeTLS always returns a non-nil error. After [Server.Shutdown] or // [Server.Close], the returned error is [ErrServerClosed]. -func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { - if srv.shuttingDown() { +func (s *Server) ListenAndServeTLS(certFile, keyFile string) error { + if s.shuttingDown() { return ErrServerClosed } - addr := srv.Addr + addr := s.Addr if addr == "" { addr = ":https" } @@ -3555,55 +3700,61 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { defer ln.Close() - return srv.ServeTLS(ln, certFile, keyFile) + return s.ServeTLS(ln, certFile, keyFile) } // setupHTTP2_ServeTLS conditionally configures HTTP/2 on -// srv and reports whether there was an error setting it up. If it is +// s and reports whether there was an error setting it up. If it is // not configured for policy reasons, nil is returned. -func (srv *Server) setupHTTP2_ServeTLS() error { - srv.nextProtoOnce.Do(srv.onceSetNextProtoDefaults) - return srv.nextProtoErr +func (s *Server) setupHTTP2_ServeTLS() error { + s.nextProtoOnce.Do(s.onceSetNextProtoDefaults) + return s.nextProtoErr } // setupHTTP2_Serve is called from (*Server).Serve and conditionally -// configures HTTP/2 on srv using a more conservative policy than +// configures HTTP/2 on s using a more conservative policy than // setupHTTP2_ServeTLS because Serve is called after tls.Listen, // and may be called concurrently. See shouldConfigureHTTP2ForServe. // // The tests named TestTransportAutomaticHTTP2* and // TestConcurrentServerServe in server_test.go demonstrate some // of the supported use cases and motivations. -func (srv *Server) setupHTTP2_Serve() error { - srv.nextProtoOnce.Do(srv.onceSetNextProtoDefaults_Serve) - return srv.nextProtoErr +func (s *Server) setupHTTP2_Serve() error { + s.nextProtoOnce.Do(s.onceSetNextProtoDefaults_Serve) + return s.nextProtoErr } -func (srv *Server) onceSetNextProtoDefaults_Serve() { - if srv.shouldConfigureHTTP2ForServe() { - srv.onceSetNextProtoDefaults() +func (s *Server) onceSetNextProtoDefaults_Serve() { + if s.shouldConfigureHTTP2ForServe() { + s.onceSetNextProtoDefaults() } } var http2server = godebug.New("http2server") // onceSetNextProtoDefaults configures HTTP/2, if the user hasn't -// configured otherwise. (by setting srv.TLSNextProto non-nil) -// It must only be called via srv.nextProtoOnce (use srv.setupHTTP2_*). -func (srv *Server) onceSetNextProtoDefaults() { +// configured otherwise. (by setting s.TLSNextProto non-nil) +// It must only be called via s.nextProtoOnce (use s.setupHTTP2_*). +func (s *Server) onceSetNextProtoDefaults() { if omitBundledHTTP2 { return } + p := s.protocols() + if !p.HTTP2() && !p.UnencryptedHTTP2() { + return + } if http2server.Value() == "0" { http2server.IncNonDefault() return } - // Enable HTTP/2 by default if the user hasn't otherwise - // configured their TLSNextProto map. - if srv.TLSNextProto == nil { - conf := &http2Server{} - srv.nextProtoErr = http2ConfigureServer(srv, conf) + if _, ok := s.TLSNextProto["h2"]; ok { + // TLSNextProto already contains an HTTP/2 implementation. + // The user probably called golang.org/x/net/http2.ConfigureServer + // to add it. + return } + conf := &http2Server{} + s.nextProtoErr = http2ConfigureServer(s, conf) } // TimeoutHandler returns a [Handler] that runs h with the given time limit. @@ -3677,9 +3828,7 @@ func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { tw.mu.Lock() defer tw.mu.Unlock() dst := w.Header() - for k, vv := range tw.h { - dst[k] = vv - } + maps.Copy(dst, tw.h) if !tw.wroteHeader { tw.code = StatusOK } diff --git a/src/net/http/sniff_test.go b/src/net/http/sniff_test.go index d6ef4090..68c8a6af 100644 --- a/src/net/http/sniff_test.go +++ b/src/net/http/sniff_test.go @@ -10,7 +10,7 @@ import ( "io" "log" . "net/http" - "reflect" + "slices" "strconv" "strings" "testing" @@ -144,7 +144,7 @@ func testServerIssue5953(t *testing.T, mode testMode) { got := resp.Header["Content-Type"] want := []string{""} - if !reflect.DeepEqual(got, want) { + if !slices.Equal(got, want) { t.Errorf("Content-Type = %q; want %q", got, want) } resp.Body.Close() diff --git a/src/net/http/transfer.go b/src/net/http/transfer.go index 5a3c6cef..67555128 100644 --- a/src/net/http/transfer.go +++ b/src/net/http/transfer.go @@ -11,6 +11,7 @@ import ( "fmt" "internal/godebug" "io" + "maps" "net/http/httptrace" "net/http/internal" "net/http/internal/ascii" @@ -350,7 +351,7 @@ func (t *transferWriter) writeBody(w io.Writer) (err error) { // nopCloser or readTrackingBody. This is to ensure that we can take advantage of // OS-level optimizations in the event that the body is an // *os.File. - if t.Body != nil { + if !t.ResponseToHEAD && t.Body != nil { var body = t.unwrapBody() if chunked(t.TransferEncoding) { if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { @@ -392,7 +393,7 @@ func (t *transferWriter) writeBody(w io.Writer) (err error) { t.ContentLength, ncopy) } - if chunked(t.TransferEncoding) { + if !t.ResponseToHEAD && chunked(t.TransferEncoding) { // Write Trailer header if t.Trailer != nil { if err := t.Trailer.Write(w); err != nil { @@ -954,9 +955,7 @@ func mergeSetHeader(dst *Header, src Header) { *dst = src return } - for k, vv := range src { - (*dst)[k] = vv - } + maps.Copy(*dst, src) } // unreadDataSizeLocked returns the number of bytes of unread input. diff --git a/src/net/http/transport.go b/src/net/http/transport.go index da9163a2..41e67418 100644 --- a/src/net/http/transport.go +++ b/src/net/http/transport.go @@ -20,6 +20,7 @@ import ( "internal/godebug" "io" "log" + "maps" "net" "net/http/httptrace" "net/http/internal/ascii" @@ -75,8 +76,7 @@ const DefaultMaxIdleConnsPerHost = 2 // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 // for HTTPS URLs, depending on whether the server supports HTTP/2, // and how the Transport is configured. The [DefaultTransport] supports HTTP/2. -// To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 -// and call ConfigureTransport. See the package docs for more about HTTP/2. +// To explicitly enable HTTP/2 on a transport, set [Transport.Protocols]. // // Responses with status codes in the 1xx range are either handled // automatically (100 expect-continue) or ignored. The one @@ -293,6 +293,22 @@ type Transport struct { // To use a custom dialer or TLS config and still attempt HTTP/2 // upgrades, set this to true. ForceAttemptHTTP2 bool + + // HTTP2 configures HTTP/2 connections. + // + // This field does not yet have any effect. + // See https://go.dev/issue/67813. + HTTP2 *HTTP2Config + + // Protocols is the set of protocols supported by the transport. + // + // If Protocols includes UnencryptedHTTP2 and does not include HTTP1, + // the transport will use unencrypted HTTP/2 for requests for http:// URLs. + // + // If Protocols is nil, the default is usually HTTP/1 only. + // If ForceAttemptHTTP2 is true, or if TLSNextProto contains an "h2" entry, + // the default is HTTP/1 and HTTP/2. + Protocols *Protocols } func (t *Transport) writeBufferSize() int { @@ -338,10 +354,18 @@ func (t *Transport) Clone() *Transport { if t.TLSClientConfig != nil { t2.TLSClientConfig = t.TLSClientConfig.Clone() } + if t.HTTP2 != nil { + t2.HTTP2 = &HTTP2Config{} + *t2.HTTP2 = *t.HTTP2 + } + if t.Protocols != nil { + t2.Protocols = &Protocols{} + *t2.Protocols = *t.Protocols + } if !t.tlsNextProtoWasNil { - npm := map[string]func(authority string, c *tls.Conn) RoundTripper{} - for k, v := range t.TLSNextProto { - npm[k] = v + npm := maps.Clone(t.TLSNextProto) + if npm == nil { + npm = make(map[string]func(authority string, c *tls.Conn) RoundTripper) } t2.TLSNextProto = npm } @@ -388,18 +412,12 @@ func (t *Transport) onceSetNextProtoDefaults() { } } - if t.TLSNextProto != nil { - // This is the documented way to disable http2 on a - // Transport. + if _, ok := t.TLSNextProto["h2"]; ok { + // There's an existing HTTP/2 implementation installed. return } - if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()) { - // Be conservative and don't automatically enable - // http2 if they've specified a custom TLS config or - // custom dialers. Let them opt-in themselves via - // http2.ConfigureTransport so we don't surprise them - // by modifying their tls.Config. Issue 14275. - // However, if ForceAttemptHTTP2 is true, it overrides the above checks. + protocols := t.protocols() + if !protocols.HTTP2() && !protocols.UnencryptedHTTP2() { return } if omitBundledHTTP2 { @@ -426,6 +444,40 @@ func (t *Transport) onceSetNextProtoDefaults() { t2.MaxHeaderListSize = uint32(limit1) } } + + // Server.ServeTLS clones the tls.Config before modifying it. + // Transport doesn't. We may want to make the two consistent some day. + // + // http2configureTransport will have already set NextProtos, but adjust it again + // here to remove HTTP/1.1 if the user has disabled it. + t.TLSClientConfig.NextProtos = adjustNextProtos(t.TLSClientConfig.NextProtos, protocols) +} + +func (t *Transport) protocols() Protocols { + if t.Protocols != nil { + return *t.Protocols // user-configured set + } + var p Protocols + p.SetHTTP1(true) // default always includes HTTP/1 + switch { + case t.TLSNextProto != nil: + // Setting TLSNextProto to an empty map is a documented way + // to disable HTTP/2 on a Transport. + if t.TLSNextProto["h2"] != nil { + p.SetHTTP2(true) + } + case !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()): + // Be conservative and don't automatically enable + // http2 if they've specified a custom TLS config or + // custom dialers. Let them opt-in themselves via + // Transport.Protocols.SetHTTP2(true) so we don't surprise them + // by modifying their tls.Config. Issue 14275. + // However, if ForceAttemptHTTP2 is true, it overrides the above checks. + case http2client.Value() == "0": + default: + p.SetHTTP2(true) + } + return p } // ProxyFromEnvironment returns the URL of the proxy to use for a @@ -820,9 +872,9 @@ func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { if _, exists := oldMap[scheme]; exists { panic("protocol " + scheme + " already registered") } - newMap := make(map[string]RoundTripper) - for k, v := range oldMap { - newMap[k] = v + newMap := maps.Clone(oldMap) + if newMap == nil { + newMap = make(map[string]RoundTripper) } newMap[scheme] = rt t.altProto.Store(newMap) @@ -1857,6 +1909,24 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *pers } } + // Possible unencrypted HTTP/2 with prior knowledge. + unencryptedHTTP2 := pconn.tlsState == nil && + t.Protocols != nil && + t.Protocols.UnencryptedHTTP2() && + !t.Protocols.HTTP1() + if unencryptedHTTP2 { + next, ok := t.TLSNextProto[nextProtoUnencryptedHTTP2] + if !ok { + return nil, errors.New("http: Transport does not support unencrypted HTTP/2") + } + alt := next(cm.targetAddr, unencryptedTLSConn(pconn.conn)) + if e, ok := alt.(erringRoundTripper); ok { + // pconn.conn was closed by next (http2configureTransports.upgradeFn). + return nil, e.RoundTripErr() + } + return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil + } + if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { alt := next(cm.targetAddr, pconn.conn.(*tls.Conn)) @@ -2387,8 +2457,6 @@ func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTr trace.GotFirstResponseByte() } } - num1xx := 0 // number of informational 1xx headers received - const max1xxResponses = 5 // arbitrary bound on number of informational responses continueCh := rc.continueCh for { @@ -2408,15 +2476,18 @@ func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTr // treat 101 as a terminal status, see issue 26161 is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols if is1xxNonTerminal { - num1xx++ - if num1xx > max1xxResponses { - return nil, errors.New("net/http: too many 1xx informational responses") - } - pc.readLimit = pc.maxHeaderResponseSize() // reset the limit if trace != nil && trace.Got1xxResponse != nil { if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { return nil, err } + // If the 1xx response was delivered to the user, + // then they're responsible for limiting the number of + // responses. Reset the header limit. + // + // If the user didn't examine the 1xx response, then we + // limit the size of all headers (including both 1xx + // and the final response) to maxHeaderResponseSize. + pc.readLimit = pc.maxHeaderResponseSize() // reset the limit } continue } diff --git a/src/net/http/transport_test.go b/src/net/http/transport_test.go index ae7159da..a454db5e 100644 --- a/src/net/http/transport_test.go +++ b/src/net/http/transport_test.go @@ -22,6 +22,7 @@ import ( "fmt" "go/token" "internal/nettrace" + "internal/synctest" "io" "log" mrand "math/rand" @@ -36,6 +37,7 @@ import ( "os" "reflect" "runtime" + "slices" "strconv" "strings" "sync" @@ -2585,8 +2587,8 @@ func runCancelTestTransport(t *testing.T, mode testMode, f func(t *testing.T, te // runCancelTestChannel uses Request.Cancel. func runCancelTestChannel(t *testing.T, mode testMode, f func(t *testing.T, test cancelTest)) { - var cancelOnce sync.Once cancelc := make(chan struct{}) + cancelOnce := sync.OnceFunc(func() { close(cancelc) }) f(t, cancelTest{ mode: mode, newReq: func(req *Request) *Request { @@ -2594,9 +2596,7 @@ func runCancelTestChannel(t *testing.T, mode testMode, f func(t *testing.T, test return req }, cancel: func(tr *Transport, req *Request) { - cancelOnce.Do(func() { - close(cancelc) - }) + cancelOnce() }, checkErr: func(when string, err error) { if !errors.Is(err, ExportErrRequestCanceled) && !errors.Is(err, ExportErrRequestCanceledConn) { @@ -3259,29 +3259,68 @@ func testTransportIgnore1xxResponses(t *testing.T, mode testMode) { } } -func TestTransportLimits1xxResponses(t *testing.T) { - run(t, testTransportLimits1xxResponses, []testMode{http1Mode}) -} +func TestTransportLimits1xxResponses(t *testing.T) { run(t, testTransportLimits1xxResponses) } func testTransportLimits1xxResponses(t *testing.T, mode testMode) { cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - conn, buf, _ := w.(Hijacker).Hijack() + w.Header().Add("X-Header", strings.Repeat("a", 100)) for i := 0; i < 10; i++ { - buf.Write([]byte("HTTP/1.1 123 OneTwoThree\r\n\r\n")) + w.WriteHeader(123) } - buf.Write([]byte("HTTP/1.1 204 No Content\r\n\r\n")) - buf.Flush() - conn.Close() + w.WriteHeader(204) })) cst.tr.DisableKeepAlives = true // prevent log spam; our test server is hanging up anyway + cst.tr.MaxResponseHeaderBytes = 1000 res, err := cst.c.Get(cst.ts.URL) - if res != nil { - defer res.Body.Close() + if err == nil { + res.Body.Close() + t.Fatalf("RoundTrip succeeded; want error") } - got := fmt.Sprint(err) - wantSub := "too many 1xx informational responses" - if !strings.Contains(got, wantSub) { - t.Errorf("Get error = %v; want substring %q", err, wantSub) + for _, want := range []string{ + "response headers exceeded", + "too many 1xx", + "header list too large", + } { + if strings.Contains(err.Error(), want) { + return + } + } + t.Errorf(`got error %q; want "response headers exceeded" or "too many 1xx"`, err) +} + +func TestTransportDoesNotLimitDelivered1xxResponses(t *testing.T) { + run(t, testTransportDoesNotLimitDelivered1xxResponses) +} +func testTransportDoesNotLimitDelivered1xxResponses(t *testing.T, mode testMode) { + if mode == http2Mode { + t.Skip("skip until x/net/http2 updated") + } + const num1xx = 10 + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Add("X-Header", strings.Repeat("a", 100)) + for i := 0; i < 10; i++ { + w.WriteHeader(123) + } + w.WriteHeader(204) + })) + cst.tr.DisableKeepAlives = true // prevent log spam; our test server is hanging up anyway + cst.tr.MaxResponseHeaderBytes = 1000 + + got1xx := 0 + ctx := httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + Got1xxResponse: func(code int, header textproto.MIMEHeader) error { + got1xx++ + return nil + }, + }) + req, _ := NewRequestWithContext(ctx, "GET", cst.ts.URL, nil) + res, err := cst.c.Do(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if got1xx != num1xx { + t.Errorf("Got %v 1xx responses, want %x", got1xx, num1xx) } } @@ -4181,53 +4220,175 @@ func TestTransportTraceGotConnH2IdleConns(t *testing.T) { wantIdle("after round trip", 1) } -func TestTransportRemovesH2ConnsAfterIdle(t *testing.T) { - run(t, testTransportRemovesH2ConnsAfterIdle, []testMode{http2Mode}) +// https://go.dev/issue/70515 +// +// When the first request on a new connection fails, we do not retry the request. +// If the first request on a connection races with IdleConnTimeout, +// we should not fail the request. +func TestTransportIdleConnRacesRequest(t *testing.T) { + // Use unencrypted HTTP/2, since the *tls.Conn interfers with our ability to + // block the connection closing. + runSynctest(t, testTransportIdleConnRacesRequest, []testMode{http1Mode, http2UnencryptedMode}) } -func testTransportRemovesH2ConnsAfterIdle(t *testing.T, mode testMode) { +func testTransportIdleConnRacesRequest(t testing.TB, mode testMode) { + if mode == http2UnencryptedMode { + t.Skip("remove skip when #70515 is fixed") + } + timeout := 1 * time.Millisecond + trFunc := func(tr *Transport) { + tr.IdleConnTimeout = timeout + } + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + }), trFunc, optFakeNet) + cst.li.trackConns = true + + // We want to put a connection into the pool which has never had a request made on it. + // + // Make a request and cancel it before the dial completes. + // Then complete the dial. + dialc := make(chan struct{}) + cst.li.onDial = func() { + <-dialc + } + ctx, cancel := context.WithCancel(context.Background()) + req1c := make(chan error) + go func() { + req, _ := NewRequestWithContext(ctx, "GET", cst.ts.URL, nil) + resp, err := cst.c.Do(req) + if err == nil { + resp.Body.Close() + } + req1c <- err + }() + // Wait for the connection attempt to start. + synctest.Wait() + // Cancel the request. + cancel() + synctest.Wait() + if err := <-req1c; err == nil { + t.Fatal("expected request to fail, but it succeeded") + } + // Unblock the dial, placing a new, unused connection into the Transport's pool. + close(dialc) + + // We want IdleConnTimeout to race with a new request. + // + // There's no perfect way to do this, but the following exercises the bug in #70515: + // Block net.Conn.Close, wait until IdleConnTimeout occurs, and make a request while + // the connection close is still blocked. + // + // First: Wait for IdleConnTimeout. The net.Conn.Close blocks. + synctest.Wait() + closec := make(chan struct{}) + cst.li.conns[0].peer.onClose = func() { + <-closec + } + time.Sleep(timeout) + synctest.Wait() + // Make a request, which will use a new connection (since the existing one is closing). + req2c := make(chan error) + go func() { + resp, err := cst.c.Get(cst.ts.URL) + if err == nil { + resp.Body.Close() + } + req2c <- err + }() + // Don't synctest.Wait here: The HTTP/1 transport closes the idle conn + // with a mutex held, and we'll end up in a deadlock. + close(closec) + if err := <-req2c; err != nil { + t.Fatalf("Get: %v", err) + } +} + +func TestTransportRemovesConnsAfterIdle(t *testing.T) { + runSynctest(t, testTransportRemovesConnsAfterIdle) +} +func testTransportRemovesConnsAfterIdle(t testing.TB, mode testMode) { if testing.Short() { t.Skip("skipping in short mode") } - timeout := 1 * time.Millisecond - retry := true - for retry { - trFunc := func(tr *Transport) { - tr.MaxConnsPerHost = 1 - tr.MaxIdleConnsPerHost = 1 - tr.IdleConnTimeout = timeout - } - cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), trFunc) + timeout := 1 * time.Second + trFunc := func(tr *Transport) { + tr.MaxConnsPerHost = 1 + tr.MaxIdleConnsPerHost = 1 + tr.IdleConnTimeout = timeout + } + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Addr", r.RemoteAddr) + }), trFunc, optFakeNet) - retry = false - tooShort := func(err error) bool { - if err == nil || !strings.Contains(err.Error(), "use of closed network connection") { - return false - } - if !retry { - t.Helper() - t.Logf("idle conn timeout %v may be too short; retrying with longer", timeout) - timeout *= 2 - retry = true - cst.close() - } - return true - } - - if _, err := cst.c.Get(cst.ts.URL); err != nil { - if tooShort(err) { - continue - } + // makeRequest returns the local address a request was made from + // (unique for each connection). + makeRequest := func() string { + resp, err := cst.c.Get(cst.ts.URL) + if err != nil { t.Fatalf("got error: %s", err) } + resp.Body.Close() + return resp.Header.Get("X-Addr") + } - time.Sleep(10 * timeout) - if _, err := cst.c.Get(cst.ts.URL); err != nil { - if tooShort(err) { - continue - } + addr1 := makeRequest() + + time.Sleep(timeout / 2) + synctest.Wait() + addr2 := makeRequest() + if addr1 != addr2 { + t.Fatalf("two requests made within IdleConnTimeout should have used the same conn, but used %v, %v", addr1, addr2) + } + + time.Sleep(timeout) + synctest.Wait() + addr3 := makeRequest() + if addr1 == addr3 { + t.Fatalf("two requests made more than IdleConnTimeout apart should have used different conns, but used %v, %v", addr1, addr3) + } +} + +func TestTransportRemovesConnsAfterBroken(t *testing.T) { + runSynctest(t, testTransportRemovesConnsAfterBroken) +} +func testTransportRemovesConnsAfterBroken(t testing.TB, mode testMode) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + trFunc := func(tr *Transport) { + tr.MaxConnsPerHost = 1 + tr.MaxIdleConnsPerHost = 1 + } + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Addr", r.RemoteAddr) + }), trFunc, optFakeNet) + cst.li.trackConns = true + + // makeRequest returns the local address a request was made from + // (unique for each connection). + makeRequest := func() string { + resp, err := cst.c.Get(cst.ts.URL) + if err != nil { t.Fatalf("got error: %s", err) } + resp.Body.Close() + return resp.Header.Get("X-Addr") + } + + addr1 := makeRequest() + addr2 := makeRequest() + if addr1 != addr2 { + t.Fatalf("successive requests should have used the same conn, but used %v, %v", addr1, addr2) + } + + // The connection breaks. + synctest.Wait() + cst.li.conns[0].peer.Close() + synctest.Wait() + addr3 := makeRequest() + if addr1 == addr3 { + t.Fatalf("successive requests made with conn broken between should have used different conns, but used %v, %v", addr1, addr3) } } @@ -4453,7 +4614,7 @@ func TestTransportFlushesBodyChunks(t *testing.T) { "5\r\nnum2\n\r\n", "0\r\n\r\n", } - if !reflect.DeepEqual(lw.writes, want) { + if !slices.Equal(lw.writes, want) { t.Errorf("Writes differed.\n Got: %q\nWant: %q\n", lw.writes, want) } } @@ -5113,20 +5274,16 @@ func testTransportEventTraceTLSVerify(t *testing.T, mode testMode) { } } -var ( - isDNSHijackedOnce sync.Once - isDNSHijacked bool -) +var isDNSHijacked = sync.OnceValue(func() bool { + addrs, _ := net.LookupHost("dns-should-not-resolve.golang") + return len(addrs) != 0 +}) func skipIfDNSHijacked(t *testing.T) { // Skip this test if the user is using a shady/ISP // DNS server hijacking queries. // See issues 16732, 16716. - isDNSHijackedOnce.Do(func() { - addrs, _ := net.LookupHost("dns-should-not-resolve.golang") - isDNSHijacked = len(addrs) != 0 - }) - if isDNSHijacked { + if isDNSHijacked() { t.Skip("skipping; test requires non-hijacking DNS server") } } @@ -5284,7 +5441,7 @@ func testTransportMaxIdleConns(t *testing.T, mode testMode) { "|http|host-2.dns-is-faked.golang:" + port, "|http|host-3.dns-is-faked.golang:" + port, } - if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { + if got := tr.IdleConnKeysForTesting(); !slices.Equal(got, want) { t.Fatalf("idle conn keys mismatch.\n got: %q\nwant: %q\n", got, want) } @@ -5296,7 +5453,7 @@ func testTransportMaxIdleConns(t *testing.T, mode testMode) { "|http|host-3.dns-is-faked.golang:" + port, "|http|host-4.dns-is-faked.golang:" + port, } - if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { + if got := tr.IdleConnKeysForTesting(); !slices.Equal(got, want) { t.Fatalf("idle conn keys mismatch after 5th host.\n got: %q\nwant: %q\n", got, want) } } @@ -5343,7 +5500,9 @@ timeoutLoop: return false } } - res.Body.Close() + if err == nil { + res.Body.Close() + } conns := idleConns() if len(conns) != 1 { if len(conns) == 0 { @@ -5462,7 +5621,7 @@ func TestTransportReturnsPeekError(t *testing.T) { errValue := errors.New("specific error value") wrote := make(chan struct{}) - var wroteOnce sync.Once + wroteOnce := sync.OnceFunc(func() { close(wrote) }) tr := &Transport{ Dial: func(network, addr string) (net.Conn, error) { @@ -5472,7 +5631,7 @@ func TestTransportReturnsPeekError(t *testing.T) { return 0, errValue }, write: func(p []byte) (int, error) { - wroteOnce.Do(func() { close(wrote) }) + wroteOnce() return len(p), nil }, } @@ -6327,12 +6486,16 @@ func TestTransportClone(t *testing.T) { GetProxyConnectHeader: func(context.Context, *url.URL, string) (Header, error) { return nil, nil }, MaxResponseHeaderBytes: 1, ForceAttemptHTTP2: true, + HTTP2: &HTTP2Config{MaxConcurrentStreams: 1}, + Protocols: &Protocols{}, TLSNextProto: map[string]func(authority string, c *tls.Conn) RoundTripper{ "foo": func(authority string, c *tls.Conn) RoundTripper { panic("") }, }, ReadBufferSize: 1, WriteBufferSize: 1, } + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) tr2 := tr.Clone() rv := reflect.ValueOf(tr2).Elem() rt := rv.Type() @@ -7133,3 +7296,266 @@ func testValidateClientRequestTrailers(t *testing.T, mode testMode) { }) } } + +func TestTransportServerProtocols(t *testing.T) { + CondSkipHTTP2(t) + DefaultTransport.(*Transport).CloseIdleConnections() + + cert, err := tls.X509KeyPair(testcert.LocalhostCert, testcert.LocalhostKey) + if err != nil { + t.Fatal(err) + } + leafCert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + certpool := x509.NewCertPool() + certpool.AddCert(leafCert) + + for _, test := range []struct { + name string + scheme string + setup func(t *testing.T) + transport func(*Transport) + server func(*Server) + want string + }{{ + name: "http default", + scheme: "http", + want: "HTTP/1.1", + }, { + name: "https default", + scheme: "https", + transport: func(tr *Transport) { + // Transport default is HTTP/1. + }, + want: "HTTP/1.1", + }, { + name: "https transport protocols include HTTP2", + scheme: "https", + transport: func(tr *Transport) { + // Server default is to support HTTP/2, so if the Transport enables + // HTTP/2 we get it. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "https transport protocols only include HTTP1", + scheme: "https", + transport: func(tr *Transport) { + // Explicitly enable only HTTP/1. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + }, + want: "HTTP/1.1", + }, { + name: "https transport ForceAttemptHTTP2", + scheme: "https", + transport: func(tr *Transport) { + // Pre-Protocols-field way of enabling HTTP/2. + tr.ForceAttemptHTTP2 = true + }, + want: "HTTP/2.0", + }, { + name: "https transport protocols override TLSNextProto", + scheme: "https", + transport: func(tr *Transport) { + // Setting TLSNextProto to an empty map is the historical way + // of disabling HTTP/2. Explicitly enabling HTTP2 in the Protocols + // field takes precedence. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + tr.TLSNextProto = map[string]func(string, *tls.Conn) RoundTripper{} + }, + want: "HTTP/2.0", + }, { + name: "https server disables HTTP2 with TLSNextProto", + scheme: "https", + server: func(srv *Server) { + // Disable HTTP/2 on the server with TLSNextProto, + // use default Protocols value. + srv.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){} + }, + want: "HTTP/1.1", + }, { + name: "https server Protocols overrides empty TLSNextProto", + scheme: "https", + server: func(srv *Server) { + // Explicitly enabling HTTP2 in the Protocols field takes precedence + // over setting an empty TLSNextProto. + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + srv.Protocols.SetHTTP2(true) + srv.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){} + }, + want: "HTTP/2.0", + }, { + name: "https server protocols only include HTTP1", + scheme: "https", + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + }, + want: "HTTP/1.1", + }, { + name: "https server protocols include HTTP2", + scheme: "https", + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + srv.Protocols.SetHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "GODEBUG disables HTTP2 client", + scheme: "https", + setup: func(t *testing.T) { + t.Setenv("GODEBUG", "http2client=0") + }, + transport: func(tr *Transport) { + // Server default is to support HTTP/2, so if the Transport enables + // HTTP/2 we get it. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + }, + want: "HTTP/1.1", + }, { + name: "GODEBUG disables HTTP2 server", + scheme: "https", + setup: func(t *testing.T) { + t.Setenv("GODEBUG", "http2server=0") + }, + transport: func(tr *Transport) { + // Server default is to support HTTP/2, so if the Transport enables + // HTTP/2 we get it. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + }, + want: "HTTP/1.1", + }, { + name: "unencrypted HTTP2 with prior knowledge", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetUnencryptedHTTP2(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + srv.Protocols.SetUnencryptedHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "unencrypted HTTP2 only on server", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetUnencryptedHTTP2(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetUnencryptedHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "unencrypted HTTP2 with no server support", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetUnencryptedHTTP2(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + }, + want: "error", + }, { + name: "HTTP1 with no server support", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetUnencryptedHTTP2(true) + }, + want: "error", + }, { + name: "HTTPS1 with no server support", + scheme: "https", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP2(true) + }, + want: "error", + }} { + t.Run(test.name, func(t *testing.T) { + // We don't use httptest here because it makes its own decisions + // about how to enable/disable HTTP/2. + srv := &Server{ + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + }, + Handler: HandlerFunc(func(w ResponseWriter, req *Request) { + w.Header().Set("X-Proto", req.Proto) + }), + } + tr := &Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + }, + } + + if test.setup != nil { + test.setup(t) + } + if test.server != nil { + test.server(srv) + } + if test.transport != nil { + test.transport(tr) + } else { + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + } + + listener := newLocalListener(t) + srvc := make(chan error, 1) + go func() { + switch test.scheme { + case "http": + srvc <- srv.Serve(listener) + case "https": + srvc <- srv.ServeTLS(listener, "", "") + } + }() + t.Cleanup(func() { + srv.Close() + <-srvc + }) + + client := &Client{Transport: tr} + resp, err := client.Get(test.scheme + "://" + listener.Addr().String()) + if err != nil { + if test.want == "error" { + return + } + t.Fatal(err) + } + if got := resp.Header.Get("X-Proto"); got != test.want { + t.Fatalf("request proto %q, want %q", got, test.want) + } + }) + } +} diff --git a/src/net/interface_linux.go b/src/net/interface_linux.go index 9112ecc8..7856dae8 100644 --- a/src/net/interface_linux.go +++ b/src/net/interface_linux.go @@ -129,22 +129,14 @@ func interfaceAddrTable(ifi *Interface) ([]Addr, error) { if err != nil { return nil, os.NewSyscallError("parsenetlinkmessage", err) } - var ift []Interface - if ifi == nil { - var err error - ift, err = interfaceTable(0) - if err != nil { - return nil, err - } - } - ifat, err := addrTable(ift, ifi, msgs) + ifat, err := addrTable(ifi, msgs) if err != nil { return nil, err } return ifat, nil } -func addrTable(ift []Interface, ifi *Interface, msgs []syscall.NetlinkMessage) ([]Addr, error) { +func addrTable(ifi *Interface, msgs []syscall.NetlinkMessage) ([]Addr, error) { var ifat []Addr loop: for _, m := range msgs { @@ -153,14 +145,7 @@ loop: break loop case syscall.RTM_NEWADDR: ifam := (*syscall.IfAddrmsg)(unsafe.Pointer(&m.Data[0])) - if len(ift) != 0 || ifi.Index == int(ifam.Index) { - if len(ift) != 0 { - var err error - ifi, err = interfaceByIndex(ift, int(ifam.Index)) - if err != nil { - return nil, err - } - } + if ifi == nil || ifi.Index == int(ifam.Index) { attrs, err := syscall.ParseNetlinkRouteAttr(&m) if err != nil { return nil, os.NewSyscallError("parsenetlinkrouteattr", err) diff --git a/src/net/internal/cgotest/resstate.go b/src/net/internal/cgotest/resstate.go index 1b487110..62cfa24e 100644 --- a/src/net/internal/cgotest/resstate.go +++ b/src/net/internal/cgotest/resstate.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build cgo && darwin +//go:build !netgo && cgo && darwin package cgotest diff --git a/src/net/ip.go b/src/net/ip.go index 3e0e85e1..e3ee6ca7 100644 --- a/src/net/ip.go +++ b/src/net/ip.go @@ -301,11 +301,18 @@ func (ip IP) String() string { if len(ip) != IPv4len && len(ip) != IPv6len { return "?" + hexString(ip) } - // If IPv4, use dotted notation. - if p4 := ip.To4(); len(p4) == IPv4len { - return netip.AddrFrom4([4]byte(p4)).String() + + var buf []byte + switch len(ip) { + case IPv4len: + const maxCap = len("255.255.255.255") + buf = make([]byte, 0, maxCap) + case IPv6len: + const maxCap = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff") + buf = make([]byte, 0, maxCap) } - return netip.AddrFrom16([16]byte(ip)).String() + buf = ip.appendTo(buf) + return string(buf) } func hexString(b []byte) string { @@ -325,17 +332,41 @@ func ipEmptyString(ip IP) string { return ip.String() } +// appendTo appends the string representation of ip to b and returns the expanded b +// If len(ip) != IPv4len or IPv6len, it appends nothing. +func (ip IP) appendTo(b []byte) []byte { + // If IPv4, use dotted notation. + if p4 := ip.To4(); len(p4) == IPv4len { + ip = p4 + } + addr, _ := netip.AddrFromSlice(ip) + return addr.AppendTo(b) +} + +// AppendText implements the [encoding.TextAppender] interface. +// The encoding is the same as returned by [IP.String], with one exception: +// When len(ip) is zero, it appends nothing. +func (ip IP) AppendText(b []byte) ([]byte, error) { + if len(ip) == 0 { + return b, nil + } + if len(ip) != IPv4len && len(ip) != IPv6len { + return b, &AddrError{Err: "invalid IP address", Addr: hexString(ip)} + } + + return ip.appendTo(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface. // The encoding is the same as returned by [IP.String], with one exception: // When len(ip) is zero, it returns an empty slice. func (ip IP) MarshalText() ([]byte, error) { - if len(ip) == 0 { - return []byte(""), nil + // 24 is satisfied with all IPv4 addresses and short IPv6 addresses + b, err := ip.AppendText(make([]byte, 0, 24)) + if err != nil { + return nil, err } - if len(ip) != IPv4len && len(ip) != IPv6len { - return nil, &AddrError{Err: "invalid IP address", Addr: hexString(ip)} - } - return []byte(ip.String()), nil + return b, nil } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/net/ip_test.go b/src/net/ip_test.go index 11c0b752..55c66fdf 100644 --- a/src/net/ip_test.go +++ b/src/net/ip_test.go @@ -149,6 +149,15 @@ func TestMarshalEmptyIP(t *testing.T) { if !reflect.DeepEqual(got, []byte("")) { t.Errorf(`got %#v, want []byte("")`, got) } + + buf := make([]byte, 4) + got, err = ip.AppendText(buf) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []byte("\x00\x00\x00\x00")) { + t.Errorf(`got %#v, want []byte("\x00\x00\x00\x00")`, got) + } } var ipStringTests = []*struct { @@ -266,9 +275,54 @@ func TestIPString(t *testing.T) { if out, err := tt.in.MarshalText(); !bytes.Equal(out, tt.byt) || !reflect.DeepEqual(err, tt.error) { t.Errorf("IP.MarshalText(%v) = %v, %v, want %v, %v", tt.in, out, err, tt.byt, tt.error) } + buf := make([]byte, 4, 32) + if out, err := tt.in.AppendText(buf); !bytes.Equal(out[4:], tt.byt) || !reflect.DeepEqual(err, tt.error) { + t.Errorf("IP.AppendText(%v) = %v, %v, want %v, %v", tt.in, out[4:], err, tt.byt, tt.error) + } } } +func TestIPAppendTextNoAllocs(t *testing.T) { + // except the invalid IP + for _, tt := range ipStringTests[:len(ipStringTests)-1] { + allocs := int(testing.AllocsPerRun(1000, func() { + buf := make([]byte, 0, 64) + _, _ = tt.in.AppendText(buf) + })) + if allocs != 0 { + t.Errorf("IP(%q) AppendText allocs: %d times, want 0", tt.in, allocs) + } + } +} + +func BenchmarkIPMarshalText(b *testing.B) { + b.Run("IPv4", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + ip := IP{192, 0, 2, 1} + for range b.N { + _, _ = ip.MarshalText() + } + }) + b.Run("IPv6", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + ip := IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0xa, 0, 0xb, 0, 0xc, 0, 0xd} + for range b.N { + _, _ = ip.MarshalText() + } + }) + b.Run("IPv6_long", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + // fd7a:115c:a1e0:ab12:4843:cd96:626b:430b + ip := IP{253, 122, 17, 92, 161, 224, 171, 18, 72, 67, 205, 150, 98, 107, 67, 11} + for range b.N { + _, _ = ip.MarshalText() + } + }) +} + var sink string func BenchmarkIPString(b *testing.B) { diff --git a/src/net/lookup.go b/src/net/lookup.go index b04dfa23..f94fd8ce 100644 --- a/src/net/lookup.go +++ b/src/net/lookup.go @@ -614,6 +614,9 @@ func (r *Resolver) LookupNS(ctx context.Context, name string) ([]*NS, error) { // LookupTXT returns the DNS TXT records for the given domain name. // +// If a DNS TXT record holds multiple strings, they are concatenated as a +// single string. +// // LookupTXT uses [context.Background] internally; to specify the context, use // [Resolver.LookupTXT]. func LookupTXT(name string) ([]string, error) { @@ -621,6 +624,9 @@ func LookupTXT(name string) ([]string, error) { } // LookupTXT returns the DNS TXT records for the given domain name. +// +// If a DNS TXT record holds multiple strings, they are concatenated as a +// single string. func (r *Resolver) LookupTXT(ctx context.Context, name string) ([]string, error) { return r.lookupTXT(ctx, name) } diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go index 7052f3c9..514cbd09 100644 --- a/src/net/lookup_test.go +++ b/src/net/lookup_test.go @@ -246,14 +246,10 @@ func TestLookupGmailTXT(t *testing.T) { if len(txts) == 0 { t.Error("got no record") } - found := false - for _, txt := range txts { - if strings.Contains(txt, tt.txt) && (strings.HasSuffix(txt, tt.host) || strings.HasSuffix(txt, tt.host+".")) { - found = true - break - } - } - if !found { + + if !slices.ContainsFunc(txts, func(txt string) bool { + return strings.Contains(txt, tt.txt) && (strings.HasSuffix(txt, tt.host) || strings.HasSuffix(txt, tt.host+".")) + }) { t.Errorf("got %v; want a record containing %s, %s", txts, tt.txt, tt.host) } } @@ -302,14 +298,7 @@ func TestLookupIPv6LinkLocalAddr(t *testing.T) { if err != nil { t.Fatal(err) } - found := false - for _, addr := range addrs { - if addr == "fe80::1%lo0" { - found = true - break - } - } - if !found { + if !slices.Contains(addrs, "fe80::1%lo0") { t.Skipf("not supported on %s", runtime.GOOS) } if _, err := LookupAddr("fe80::1%lo0"); err != nil { @@ -433,7 +422,7 @@ func TestLookupLongTXT(t *testing.T) { strings.Repeat("abcdefghijklmnopqrstuvwxyABCDEFGHJIKLMNOPQRSTUVWXY", 10), "gophers rule", } - if !reflect.DeepEqual(txts, want) { + if !slices.Equal(txts, want) { t.Fatalf("LookupTXT golang.rsc.io incorrect\nhave %q\nwant %q", txts, want) } } diff --git a/src/net/lookup_windows_test.go b/src/net/lookup_windows_test.go index 8f6e4b23..4f287901 100644 --- a/src/net/lookup_windows_test.go +++ b/src/net/lookup_windows_test.go @@ -144,7 +144,7 @@ func TestNSLookupTXT(t *testing.T) { } slices.Sort(expected) slices.Sort(txt) - if !reflect.DeepEqual(expected, txt) { + if !slices.Equal(expected, txt) { t.Errorf("different results %s:\texp:%v\tgot:%v", server, toJson(expected), toJson(txt)) } }) @@ -170,7 +170,7 @@ func TestLookupLocalPTR(t *testing.T) { } slices.Sort(expected) slices.Sort(names) - if !reflect.DeepEqual(expected, names) { + if !slices.Equal(expected, names) { t.Errorf("different results %s:\texp:%v\tgot:%v", addr, toJson(expected), toJson(names)) } } @@ -201,7 +201,7 @@ func TestLookupPTR(t *testing.T) { } slices.Sort(expected) slices.Sort(names) - if !reflect.DeepEqual(expected, names) { + if !slices.Equal(expected, names) { t.Errorf("different results %s:\texp:%v\tgot:%v", addr, toJson(expected), toJson(names)) } } diff --git a/src/net/mail/example_test.go b/src/net/mail/example_test.go index d325dc79..9fadda24 100644 --- a/src/net/mail/example_test.go +++ b/src/net/mail/example_test.go @@ -10,6 +10,7 @@ import ( "log" "net/mail" "strings" + "time" ) func ExampleParseAddressList() { @@ -75,3 +76,17 @@ Message body // Subject: Gophers at Gophercon // Message body } + +func ExampleParseDate() { + dateStr := "Wed, 09 Oct 2024 09:55:06 -0700" + + t, err := mail.ParseDate(dateStr) + if err != nil { + log.Fatalf("Failed to parse date: %v", err) + } + + fmt.Println(t.Format(time.RFC3339)) + + // Output: + // 2024-10-09T09:55:06-07:00 +} diff --git a/src/net/mail/message_test.go b/src/net/mail/message_test.go index 012d51c3..dad9c367 100644 --- a/src/net/mail/message_test.go +++ b/src/net/mail/message_test.go @@ -9,6 +9,7 @@ import ( "io" "mime" "reflect" + "slices" "strings" "testing" "time" @@ -115,7 +116,7 @@ func headerEq(a, b Header) bool { if !ok { return false } - if !reflect.DeepEqual(as, bs) { + if !slices.Equal(as, bs) { return false } } diff --git a/src/net/mptcpsock_linux.go b/src/net/mptcpsock_linux.go index b2ac3ee7..42230904 100644 --- a/src/net/mptcpsock_linux.go +++ b/src/net/mptcpsock_linux.go @@ -16,7 +16,7 @@ import ( var ( mptcpOnce sync.Once mptcpAvailable bool - hasSOLMPTCP bool + hasSOLMPTCP bool // only valid if mptcpAvailable is true ) // These constants aren't in the syscall package, which is frozen @@ -34,10 +34,17 @@ func supportsMultipathTCP() bool { // Check that MPTCP is supported by attempting to create an MPTCP socket and by // looking at the returned error if any. func initMPTCPavailable() { - s, err := sysSocket(syscall.AF_INET, syscall.SOCK_STREAM, _IPPROTO_MPTCP) + family := syscall.AF_INET + if !supportsIPv4() { + family = syscall.AF_INET6 + } + s, err := sysSocket(family, syscall.SOCK_STREAM, _IPPROTO_MPTCP) + switch { case errors.Is(err, syscall.EPROTONOSUPPORT): // Not supported: >= v5.6 + return case errors.Is(err, syscall.EINVAL): // Not supported: < v5.6 + return case err == nil: // Supported and no error poll.CloseFunc(s) fallthrough @@ -119,6 +126,10 @@ func isUsingMPTCPProto(fd *netFD) bool { // Please look at the description of hasFallenBack (kernel >=5.16) and // isUsingMPTCPProto methods for more details about what is being checked here. func isUsingMultipathTCP(fd *netFD) bool { + if !supportsMultipathTCP() { + return false + } + if hasSOLMPTCP { return !hasFallenBack(fd) } diff --git a/src/net/net.go b/src/net/net.go index f8b5834a..82dc222d 100644 --- a/src/net/net.go +++ b/src/net/net.go @@ -68,6 +68,11 @@ GODEBUG environment variable (see package runtime) to go or cgo, as in: The decision can also be forced while building the Go source tree by setting the netgo or netcgo build tag. +The netgo build tag disables entirely the use of the native (CGO) resolver, +meaning the Go resolver is the only one that can be used. +With the netcgo build tag the native and the pure Go resolver are compiled into the binary, +but the native (CGO) resolver is preferred over the Go resolver. +With netcgo, the Go resolver can still be forced at runtime with GODEBUG=netdns=go. A numeric netdns setting, as in GODEBUG=netdns=1, causes the resolver to print debugging information about its decisions. diff --git a/src/net/netip/netip.go b/src/net/netip/netip.go index a1e93cb2..35abfd32 100644 --- a/src/net/netip/netip.go +++ b/src/net/netip/netip.go @@ -102,8 +102,8 @@ func AddrFrom4(addr [4]byte) Addr { func AddrFrom16(addr [16]byte) Addr { return Addr{ addr: uint128{ - byteorder.BeUint64(addr[:8]), - byteorder.BeUint64(addr[8:]), + byteorder.BEUint64(addr[:8]), + byteorder.BEUint64(addr[8:]), }, z: z6noz, } @@ -462,7 +462,9 @@ func (ip Addr) Is4() bool { return ip.z == z4 } -// Is4In6 reports whether ip is an IPv4-mapped IPv6 address. +// Is4In6 reports whether ip is an "IPv4-mapped IPv6 address" +// as defined by RFC 4291. +// That is, it reports whether ip is in ::ffff:0:0/96. func (ip Addr) Is4In6() bool { return ip.Is6() && ip.addr.hi == 0 && ip.addr.lo>>32 == 0xffff } @@ -700,8 +702,8 @@ func (ip Addr) Prefix(b int) (Prefix, error) { // [Addr.Zone] method to get it). // The ip zero value returns all zeroes. func (ip Addr) As16() (a16 [16]byte) { - byteorder.BePutUint64(a16[:8], ip.addr.hi) - byteorder.BePutUint64(a16[8:], ip.addr.lo) + byteorder.BEPutUint64(a16[:8], ip.addr.hi) + byteorder.BEPutUint64(a16[8:], ip.addr.lo) return a16 } @@ -710,7 +712,7 @@ func (ip Addr) As16() (a16 [16]byte) { // Note that 0.0.0.0 is not the zero Addr. func (ip Addr) As4() (a4 [4]byte) { if ip.z == z4 || ip.Is4In6() { - byteorder.BePutUint32(a4[:], uint32(ip.addr.lo)) + byteorder.BEPutUint32(a4[:], uint32(ip.addr.lo)) return a4 } if ip.z == z0 { @@ -726,12 +728,12 @@ func (ip Addr) AsSlice() []byte { return nil case z4: var ret [4]byte - byteorder.BePutUint32(ret[:], uint32(ip.addr.lo)) + byteorder.BEPutUint32(ret[:], uint32(ip.addr.lo)) return ret[:] default: var ret [16]byte - byteorder.BePutUint64(ret[:8], ip.addr.hi) - byteorder.BePutUint64(ret[8:], ip.addr.lo) + byteorder.BEPutUint64(ret[:8], ip.addr.hi) + byteorder.BEPutUint64(ret[8:], ip.addr.lo) return ret[:] } } @@ -966,27 +968,32 @@ func (ip Addr) StringExpanded() string { return string(ret) } +// AppendText implements the [encoding.TextAppender] interface, +// It is the same as [Addr.AppendTo]. +func (ip Addr) AppendText(b []byte) ([]byte, error) { + return ip.AppendTo(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface, // The encoding is the same as returned by [Addr.String], with one exception: // If ip is the zero [Addr], the encoding is the empty string. func (ip Addr) MarshalText() ([]byte, error) { + buf := []byte{} switch ip.z { case z0: - return []byte(""), nil case z4: - max := len("255.255.255.255") - b := make([]byte, 0, max) - return ip.appendTo4(b), nil + const maxCap = len("255.255.255.255") + buf = make([]byte, 0, maxCap) default: if ip.Is4In6() { - max := len("::ffff:255.255.255.255%enp5s0") - b := make([]byte, 0, max) - return ip.appendTo4In6(b), nil + const maxCap = len("::ffff:255.255.255.255%enp5s0") + buf = make([]byte, 0, maxCap) + break } - max := len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") - b := make([]byte, 0, max) - return ip.appendTo6(b), nil + const maxCap = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") + buf = make([]byte, 0, maxCap) } + return ip.AppendText(buf) } // UnmarshalText implements the encoding.TextUnmarshaler interface. @@ -1004,22 +1011,29 @@ func (ip *Addr) UnmarshalText(text []byte) error { return err } -func (ip Addr) marshalBinaryWithTrailingBytes(trailingBytes int) []byte { - var b []byte +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (ip Addr) AppendBinary(b []byte) ([]byte, error) { switch ip.z { case z0: - b = make([]byte, trailingBytes) case z4: - b = make([]byte, 4+trailingBytes) - byteorder.BePutUint32(b, uint32(ip.addr.lo)) + b = byteorder.BEAppendUint32(b, uint32(ip.addr.lo)) default: - z := ip.Zone() - b = make([]byte, 16+len(z)+trailingBytes) - byteorder.BePutUint64(b[:8], ip.addr.hi) - byteorder.BePutUint64(b[8:], ip.addr.lo) - copy(b[16:], z) + b = byteorder.BEAppendUint64(b, ip.addr.hi) + b = byteorder.BEAppendUint64(b, ip.addr.lo) + b = append(b, ip.Zone()...) + } + return b, nil +} + +func (ip Addr) marshalBinarySize() int { + switch ip.z { + case z0: + return 0 + case z4: + return 4 + default: + return 16 + len(ip.Zone()) } - return b } // MarshalBinary implements the [encoding.BinaryMarshaler] interface. @@ -1027,7 +1041,7 @@ func (ip Addr) marshalBinaryWithTrailingBytes(trailingBytes int) []byte { // the 4-byte form for an IPv4 address, // and the 16-byte form with zone appended for an IPv6 address. func (ip Addr) MarshalBinary() ([]byte, error) { - return ip.marshalBinaryWithTrailingBytes(0), nil + return ip.AppendBinary(make([]byte, 0, ip.marshalBinarySize())) } // UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. @@ -1198,21 +1212,27 @@ func (p AddrPort) AppendTo(b []byte) []byte { return b } +// AppendText implements the [encoding.TextAppender] interface. The +// encoding is the same as returned by [AddrPort.AppendTo]. +func (p AddrPort) AppendText(b []byte) ([]byte, error) { + return p.AppendTo(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface. The // encoding is the same as returned by [AddrPort.String], with one exception: if // p.Addr() is the zero [Addr], the encoding is the empty string. func (p AddrPort) MarshalText() ([]byte, error) { - var max int + buf := []byte{} switch p.ip.z { case z0: case z4: - max = len("255.255.255.255:65535") + const maxCap = len("255.255.255.255:65535") + buf = make([]byte, 0, maxCap) default: - max = len("[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535") + const maxCap = len("[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535") + buf = make([]byte, 0, maxCap) } - b := make([]byte, 0, max) - b = p.AppendTo(b) - return b, nil + return p.AppendText(buf) } // UnmarshalText implements the encoding.TextUnmarshaler @@ -1228,13 +1248,22 @@ func (p *AddrPort) UnmarshalText(text []byte) error { return err } +// AppendBinary implements the [encoding.BinaryAppendler] interface. +// It returns [Addr.AppendBinary] with an additional two bytes appended +// containing the port in little-endian. +func (p AddrPort) AppendBinary(b []byte) ([]byte, error) { + b, err := p.Addr().AppendBinary(b) + if err != nil { + return nil, err + } + return byteorder.LEAppendUint16(b, p.Port()), nil +} + // MarshalBinary implements the [encoding.BinaryMarshaler] interface. // It returns [Addr.MarshalBinary] with an additional two bytes appended // containing the port in little-endian. func (p AddrPort) MarshalBinary() ([]byte, error) { - b := p.Addr().marshalBinaryWithTrailingBytes(2) - byteorder.LePutUint16(b[len(b)-2:], p.Port()) - return b, nil + return p.AppendBinary(make([]byte, 0, p.Addr().marshalBinarySize()+2)) } // UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. @@ -1248,7 +1277,7 @@ func (p *AddrPort) UnmarshalBinary(b []byte) error { if err != nil { return err } - *p = AddrPortFrom(addr, byteorder.LeUint16(b[len(b)-2:])) + *p = AddrPortFrom(addr, byteorder.LEUint16(b[len(b)-2:])) return nil } @@ -1487,21 +1516,27 @@ func (p Prefix) AppendTo(b []byte) []byte { return b } +// AppendText implements the [encoding.TextAppender] interface. +// It is the same as [Prefix.AppendTo]. +func (p Prefix) AppendText(b []byte) ([]byte, error) { + return p.AppendTo(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface, // The encoding is the same as returned by [Prefix.String], with one exception: // If p is the zero value, the encoding is the empty string. func (p Prefix) MarshalText() ([]byte, error) { - var max int + buf := []byte{} switch p.ip.z { case z0: case z4: - max = len("255.255.255.255/32") + const maxCap = len("255.255.255.255/32") + buf = make([]byte, 0, maxCap) default: - max = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0/128") + const maxCap = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0/128") + buf = make([]byte, 0, maxCap) } - b := make([]byte, 0, max) - b = p.AppendTo(b) - return b, nil + return p.AppendText(buf) } // UnmarshalText implements the encoding.TextUnmarshaler interface. @@ -1517,13 +1552,23 @@ func (p *Prefix) UnmarshalText(text []byte) error { return err } +// AppendBinary implements the [encoding.AppendMarshaler] interface. +// It returns [Addr.AppendBinary] with an additional byte appended +// containing the prefix bits. +func (p Prefix) AppendBinary(b []byte) ([]byte, error) { + b, err := p.Addr().withoutZone().AppendBinary(b) + if err != nil { + return nil, err + } + return append(b, uint8(p.Bits())), nil +} + // MarshalBinary implements the [encoding.BinaryMarshaler] interface. // It returns [Addr.MarshalBinary] with an additional byte appended // containing the prefix bits. func (p Prefix) MarshalBinary() ([]byte, error) { - b := p.Addr().withoutZone().marshalBinaryWithTrailingBytes(1) - b[len(b)-1] = uint8(p.Bits()) - return b, nil + // without the zone the max length is 16, plus an additional byte is 17 + return p.AppendBinary(make([]byte, 0, p.Addr().withoutZone().marshalBinarySize()+1)) } // UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. diff --git a/src/net/netip/netip_test.go b/src/net/netip/netip_test.go index e1a0a83f..ea03f9a9 100644 --- a/src/net/netip/netip_test.go +++ b/src/net/netip/netip_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "flag" "fmt" + "internal/asan" "internal/testenv" "net" . "net/netip" @@ -351,6 +352,32 @@ func TestIPv4Constructors(t *testing.T) { } } +func TestAddrAppendText(t *testing.T) { + tests := []struct { + ip Addr + want string + }{ + {Addr{}, ""}, // zero IP + {mustIP("1.2.3.4"), "1.2.3.4"}, + {mustIP("fd7a:115c:a1e0:ab12:4843:cd96:626b:430b"), "fd7a:115c:a1e0:ab12:4843:cd96:626b:430b"}, + {mustIP("::ffff:192.168.140.255"), "::ffff:192.168.140.255"}, + {mustIP("::ffff:192.168.140.255%en0"), "::ffff:192.168.140.255%en0"}, + } + for i, tc := range tests { + ip := tc.ip + + mtAppend := make([]byte, 4, 32) + mtAppend, err := ip.AppendText(mtAppend) + mtAppend = mtAppend[4:] + if err != nil { + t.Fatal(err) + } + if string(mtAppend) != tc.want { + t.Errorf("%d. for (%v) AppendText = %q; want %q", i, ip, mtAppend, tc.want) + } + } +} + func TestAddrMarshalUnmarshalBinary(t *testing.T) { tests := []struct { ip string @@ -381,6 +408,23 @@ func TestAddrMarshalUnmarshalBinary(t *testing.T) { if ip != ip2 { t.Fatalf("got %v; want %v", ip2, ip) } + + bAppend := make([]byte, 4, 32) + bAppend, err = ip.AppendBinary(bAppend) + bAppend = bAppend[4:] + if err != nil { + t.Fatal(err) + } + if len(bAppend) != tc.wantSize { + t.Fatalf("%q encoded to size %d; want %d", tc.ip, len(bAppend), tc.wantSize) + } + var ip3 Addr + if err := ip3.UnmarshalBinary(bAppend); err != nil { + t.Fatal(err) + } + if ip != ip3 { + t.Fatalf("got %v; want %v", ip3, ip) + } } // Cannot unmarshal from unexpected IP length. @@ -416,6 +460,17 @@ func TestAddrPortMarshalTextString(t *testing.T) { if string(mt) != tt.want { t.Errorf("%d. for (%v, %v) MarshalText = %q; want %q", i, tt.in.Addr(), tt.in.Port(), mt, tt.want) } + + mtAppend := make([]byte, 4, 32) + mtAppend, err = tt.in.AppendText(mtAppend) + mtAppend = mtAppend[4:] + if err != nil { + t.Errorf("%d. for (%v, %v) AppendText error: %v", i, tt.in.Addr(), tt.in.Port(), err) + continue + } + if string(mtAppend) != tt.want { + t.Errorf("%d. for (%v, %v) AppendText = %q; want %q", i, tt.in.Addr(), tt.in.Port(), mtAppend, tt.want) + } } } @@ -448,6 +503,23 @@ func TestAddrPortMarshalUnmarshalBinary(t *testing.T) { if ipport != ipport2 { t.Fatalf("got %v; want %v", ipport2, ipport) } + + bAppend := make([]byte, 4, 32) + bAppend, err = ipport.AppendBinary(bAppend) + bAppend = bAppend[4:] + if err != nil { + t.Fatal(err) + } + if len(bAppend) != tc.wantSize { + t.Fatalf("%q encoded to size %d; want %d", tc.ipport, len(bAppend), tc.wantSize) + } + var ipport3 AddrPort + if err := ipport3.UnmarshalBinary(bAppend); err != nil { + t.Fatal(err) + } + if ipport != ipport3 { + t.Fatalf("got %v; want %v", ipport3, ipport) + } } // Cannot unmarshal from unexpected lengths. @@ -482,6 +554,17 @@ func TestPrefixMarshalTextString(t *testing.T) { if string(mt) != tt.want { t.Errorf("%d. for %v MarshalText = %q; want %q", i, tt.in, mt, tt.want) } + + mtAppend := make([]byte, 4, 64) + mtAppend, err = tt.in.AppendText(mtAppend) + mtAppend = mtAppend[4:] + if err != nil { + t.Errorf("%d. for %v AppendText error: %v", i, tt.in, err) + continue + } + if string(mtAppend) != tt.want { + t.Errorf("%d. for %v AppendText = %q; want %q", i, tt.in, mtAppend, tt.want) + } } } @@ -515,6 +598,23 @@ func TestPrefixMarshalUnmarshalBinary(t *testing.T) { if prefix != prefix2 { t.Fatalf("got %v; want %v", prefix2, prefix) } + + bAppend := make([]byte, 4, 32) + bAppend, err = prefix.AppendBinary(bAppend) + bAppend = bAppend[4:] + if err != nil { + t.Fatal(err) + } + if len(bAppend) != tc.wantSize { + t.Fatalf("%q encoded to size %d; want %d", tc.prefix, len(bAppend), tc.wantSize) + } + var prefix3 Prefix + if err := prefix3.UnmarshalBinary(bAppend); err != nil { + t.Fatal(err) + } + if prefix != prefix3 { + t.Fatalf("got %v; want %v", prefix3, prefix) + } } // Cannot unmarshal from unexpected lengths. @@ -2033,6 +2133,10 @@ var ( ) func TestNoAllocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } + // Wrappers that panic on error, to prove that our alloc-free // methods are returning successfully. panicIP := func(ip Addr, err error) Addr { @@ -2179,11 +2283,14 @@ func TestPrefixString(t *testing.T) { } } -func TestInvalidAddrPortString(t *testing.T) { +func TestAddrPortString(t *testing.T) { tests := []struct { ipp AddrPort want string }{ + {MustParseAddrPort("127.0.0.1:80"), "127.0.0.1:80"}, + {MustParseAddrPort("[0000::0]:8080"), "[::]:8080"}, + {MustParseAddrPort("[FFFF::1]:8080"), "[ffff::1]:8080"}, {AddrPort{}, "invalid AddrPort"}, {AddrPortFrom(Addr{}, 80), "invalid AddrPort"}, } diff --git a/src/net/resolverdialfunc_test.go b/src/net/resolverdialfunc_test.go index 9b45cdbc..3e8d7752 100644 --- a/src/net/resolverdialfunc_test.go +++ b/src/net/resolverdialfunc_test.go @@ -59,7 +59,7 @@ func TestResolverDialFunc(t *testing.T) { if err != nil { t.Fatal(err) } - if got, want := sortedIPStrings(ips), []string{"0:200::e00", "1.2.3.4", "1::f", "5.6.7.8"}; !reflect.DeepEqual(got, want) { + if got, want := sortedIPStrings(ips), []string{"0:200::e00", "1.2.3.4", "1::f", "5.6.7.8"}; !slices.Equal(got, want) { t.Errorf("LookupIP wrong.\n got: %q\nwant: %q\n", got, want) } }) diff --git a/src/net/sendfile_linux.go b/src/net/sendfile_linux.go index f8a7bec8..75af6174 100644 --- a/src/net/sendfile_linux.go +++ b/src/net/sendfile_linux.go @@ -20,7 +20,7 @@ const supportsSendfile = true // // if handled == false, sendFile performed no work. func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - var remain int64 = 1<<63 - 1 // by default, copy until EOF + var remain int64 = 0 // 0 indicates sending until EOF lr, ok := r.(*io.LimitedReader) if ok { diff --git a/src/net/sendfile_test.go b/src/net/sendfile_test.go index 4f341156..8f98352e 100644 --- a/src/net/sendfile_test.go +++ b/src/net/sendfile_test.go @@ -13,6 +13,7 @@ import ( "fmt" "internal/poll" "io" + "math/rand" "os" "runtime" "strconv" @@ -46,6 +47,7 @@ func expectSendfile(t *testing.T, wantConn Conn, f func()) { called bool gotHandled bool gotFD *poll.FD + gotErr error ) poll.TestHookDidSendFile = func(dstFD *poll.FD, src int, written int64, err error, handled bool) { if called { @@ -54,6 +56,7 @@ func expectSendfile(t *testing.T, wantConn Conn, f func()) { called = true gotHandled = handled gotFD = dstFD + gotErr = err } f() if !called { @@ -61,7 +64,7 @@ func expectSendfile(t *testing.T, wantConn Conn, f func()) { return } if !gotHandled { - t.Error("internal/poll.SendFile did not handle the write, want it to") + t.Error("internal/poll.SendFile did not handle the write, want it to, error:", gotErr) return } if &wantConn.(*TCPConn).fd.pfd != gotFD { @@ -69,7 +72,33 @@ func expectSendfile(t *testing.T, wantConn Conn, f func()) { } } -func TestSendfile(t *testing.T) { +func TestSendfile(t *testing.T) { testSendfile(t, newton, newtonSHA256, newtonLen, 0) } +func TestSendfileWithExactLimit(t *testing.T) { + testSendfile(t, newton, newtonSHA256, newtonLen, newtonLen) +} +func TestSendfileWithLimitLargerThanFile(t *testing.T) { + testSendfile(t, newton, newtonSHA256, newtonLen, newtonLen*2) +} +func TestSendfileWithLargeFile(t *testing.T) { + // Some platforms are not capable of handling large files with sendfile + // due to limited system resource, so we only run this test on amd64 and + // arm64 for the moment. + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("skipping on non-amd64 and non-arm64 platforms") + } + // Also skip it during short testing. + if testing.Short() { + t.Skip("Skip it during short testing") + } + + // We're using 1<<31 - 1 as the chunk size for sendfile currently, + // make an edge case file that is 1 byte bigger than that. + f := createTempFile(t, 1<<31) + // For big file like this, only verify the transmission of the file, + // skip the content check. + testSendfile(t, f.Name(), "", 1<<31, 0) +} +func testSendfile(t *testing.T, filePath, fileHash string, size, limit int64) { ln := newLocalListener(t, "tcp") defer ln.Close() @@ -87,7 +116,7 @@ func TestSendfile(t *testing.T) { defer close(errc) defer conn.Close() - f, err := os.Open(newton) + f, err := os.Open(filePath) if err != nil { errc <- err return @@ -104,7 +133,14 @@ func TestSendfile(t *testing.T) { sbytes, err = io.Copy(conn, f) default: expectSendfile(t, conn, func() { - sbytes, err = io.Copy(conn, f) + if limit > 0 { + sbytes, err = io.CopyN(conn, f, limit) + if err == io.EOF && limit > size { + err = nil + } + } else { + sbytes, err = io.Copy(conn, f) + } }) } if err != nil { @@ -112,8 +148,8 @@ func TestSendfile(t *testing.T) { return } - if sbytes != newtonLen { - errc <- fmt.Errorf("sent %d bytes; expected %d", sbytes, newtonLen) + if sbytes != size { + errc <- fmt.Errorf("sent %d bytes; expected %d", sbytes, size) return } }() @@ -133,11 +169,11 @@ func TestSendfile(t *testing.T) { t.Error(err) } - if rbytes != newtonLen { - t.Errorf("received %d bytes; expected %d", rbytes, newtonLen) + if rbytes != size { + t.Errorf("received %d bytes; expected %d", rbytes, size) } - if res := hex.EncodeToString(h.Sum(nil)); res != newtonSHA256 { + if len(fileHash) > 0 && hex.EncodeToString(h.Sum(nil)) != newtonSHA256 { t.Error("retrieved data hash did not match") } @@ -542,7 +578,7 @@ type sendFileBench struct { func (bench sendFileBench) benchSendFile(b *testing.B) { fileSize := b.N * bench.chunkSize - f := createTempFile(b, fileSize) + f := createTempFile(b, int64(fileSize)) client, server := spawnTestSocketPair(b, bench.proto) defer server.Close() @@ -568,25 +604,27 @@ func (bench sendFileBench) benchSendFile(b *testing.B) { } } -func createTempFile(b *testing.B, size int) *os.File { - f, err := os.CreateTemp(b.TempDir(), "sendfile-bench") +func createTempFile(tb testing.TB, size int64) *os.File { + f, err := os.CreateTemp(tb.TempDir(), "sendfile-bench") if err != nil { - b.Fatalf("failed to create temporary file: %v", err) + tb.Fatalf("failed to create temporary file: %v", err) } - b.Cleanup(func() { + tb.Cleanup(func() { f.Close() }) - data := make([]byte, size) - if _, err := f.Write(data); err != nil { - b.Fatalf("failed to create and feed the file: %v", err) - } - if err := f.Sync(); err != nil { - b.Fatalf("failed to save the file: %v", err) + if _, err := io.CopyN(f, newRandReader(tb), size); err != nil { + tb.Fatalf("failed to fill the file with random data: %v", err) } if _, err := f.Seek(0, io.SeekStart); err != nil { - b.Fatalf("failed to rewind the file: %v", err) + tb.Fatalf("failed to rewind the file: %v", err) } return f } + +func newRandReader(tb testing.TB) io.Reader { + seed := time.Now().UnixNano() + tb.Logf("Deterministic RNG seed based on timestamp: 0x%x", seed) + return rand.New(rand.NewSource(seed)) +} diff --git a/src/net/sendfile_unix_alt.go b/src/net/sendfile_unix_alt.go index 4056856f..db788753 100644 --- a/src/net/sendfile_unix_alt.go +++ b/src/net/sendfile_unix_alt.go @@ -9,7 +9,6 @@ package net import ( "internal/poll" "io" - "io/fs" "syscall" ) @@ -23,13 +22,7 @@ const supportsSendfile = true // // if handled == false, sendFile performed no work. func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - // Darwin, FreeBSD, DragonFly and Solaris use 0 as the "until EOF" value. - // If you pass in more bytes than the file contains, it will - // loop back to the beginning ad nauseam until it's sent - // exactly the number of bytes told to. As such, we need to - // know exactly how many bytes to send. - var remain int64 = 0 - + var remain int64 = 0 // 0 writes the entire file lr, ok := r.(*io.LimitedReader) if ok { remain, r = lr.N, lr.R @@ -39,37 +32,11 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { } // r might be an *os.File or an os.fileWithoutWriteTo. // Type assert to an interface rather than *os.File directly to handle the latter case. - f, ok := r.(interface { - fs.File - io.Seeker - syscall.Conn - }) + f, ok := r.(syscall.Conn) if !ok { return 0, nil, false } - if remain == 0 { - fi, err := f.Stat() - if err != nil { - return 0, err, false - } - if fi.Mode()&(fs.ModeSymlink|fs.ModeDevice|fs.ModeCharDevice|fs.ModeIrregular) != 0 { - return 0, nil, false - } - - remain = fi.Size() - } - - // The other quirk with Darwin/FreeBSD/DragonFly/Solaris's sendfile - // implementation is that it doesn't use the current position - // of the file -- if you pass it offset 0, it starts from - // offset 0. There's no way to tell it "start from current - // position", so we have to manage that explicitly. - pos, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err, false - } - sc, err := f.SyscallConn() if err != nil { return 0, nil, false @@ -77,7 +44,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { var werr error err = sc.Read(func(fd uintptr) bool { - written, werr, handled = poll.SendFile(&c.pfd, int(fd), pos, remain) + written, werr, handled = poll.SendFile(&c.pfd, int(fd), remain) return true }) if err == nil { @@ -88,10 +55,5 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { lr.N = remain - written } - _, err1 := f.Seek(written, io.SeekCurrent) - if err1 != nil && err == nil { - return written, err1, handled - } - return written, wrapSyscallError("sendfile", err), handled } diff --git a/src/net/smtp/smtp.go b/src/net/smtp/smtp.go index d750a285..522d80e4 100644 --- a/src/net/smtp/smtp.go +++ b/src/net/smtp/smtp.go @@ -413,9 +413,7 @@ func (c *Client) Noop() error { // Quit sends the QUIT command and closes the connection to the server. func (c *Client) Quit() error { - if err := c.hello(); err != nil { - return err - } + c.hello() // ignore error; we're quitting anyhow _, _, err := c.cmd(221, "QUIT") if err != nil { return err diff --git a/src/net/smtp/smtp_test.go b/src/net/smtp/smtp_test.go index c91c99b1..389eda9a 100644 --- a/src/net/smtp/smtp_test.go +++ b/src/net/smtp/smtp_test.go @@ -288,6 +288,37 @@ Goodbye. QUIT ` +func TestHELOFailed(t *testing.T) { + serverLines := `502 EH? +502 EH? +221 OK +` + clientLines := `EHLO localhost +HELO localhost +QUIT +` + + server := strings.Join(strings.Split(serverLines, "\n"), "\r\n") + client := strings.Join(strings.Split(clientLines, "\n"), "\r\n") + var cmdbuf strings.Builder + bcmdbuf := bufio.NewWriter(&cmdbuf) + var fake faker + fake.ReadWriter = bufio.NewReadWriter(bufio.NewReader(strings.NewReader(server)), bcmdbuf) + c := &Client{Text: textproto.NewConn(fake), localName: "localhost"} + + if err := c.Hello("localhost"); err == nil { + t.Fatal("expected EHLO to fail") + } + if err := c.Quit(); err != nil { + t.Errorf("QUIT failed: %s", err) + } + bcmdbuf.Flush() + actual := cmdbuf.String() + if client != actual { + t.Errorf("Got:\n%s\nWant:\n%s", actual, client) + } +} + func TestExtensions(t *testing.T) { fake := func(server string) (c *Client, bcmdbuf *bufio.Writer, cmdbuf *strings.Builder) { server = strings.Join(strings.Split(server, "\n"), "\r\n") diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index f5df502f..92966b70 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -324,7 +324,15 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } sd := &sysDialer{network: network, address: raddr.String()} - c, err := sd.dialTCP(context.Background(), laddr, raddr) + var ( + c *TCPConn + err error + ) + if sd.MultipathTCP() { + c, err = sd.dialMPTCP(context.Background(), laddr, raddr) + } else { + c, err = sd.dialTCP(context.Background(), laddr, raddr) + } if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } @@ -439,7 +447,15 @@ func ListenTCP(network string, laddr *TCPAddr) (*TCPListener, error) { laddr = &TCPAddr{} } sl := &sysListener{network: network, address: laddr.String()} - ln, err := sl.listenTCP(context.Background(), laddr) + var ( + ln *TCPListener + err error + ) + if sl.MultipathTCP() { + ln, err = sl.listenMPTCP(context.Background(), laddr) + } else { + ln, err = sl.listenTCP(context.Background(), laddr) + } if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } diff --git a/src/net/textproto/reader_test.go b/src/net/textproto/reader_test.go index f794879b..d510f9b3 100644 --- a/src/net/textproto/reader_test.go +++ b/src/net/textproto/reader_test.go @@ -11,6 +11,7 @@ import ( "net" "reflect" "runtime" + "slices" "strings" "sync" "testing" @@ -95,13 +96,13 @@ func TestReadDotLines(t *testing.T) { r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanother\n") s, err := r.ReadDotLines() want := []string{"dotlines", "foo", ".bar", "..baz", "quux", ""} - if !reflect.DeepEqual(s, want) || err != nil { + if !slices.Equal(s, want) || err != nil { t.Fatalf("ReadDotLines: %v, %v", s, err) } s, err = r.ReadDotLines() want = []string{"another"} - if !reflect.DeepEqual(s, want) || err != io.ErrUnexpectedEOF { + if !slices.Equal(s, want) || err != io.ErrUnexpectedEOF { t.Fatalf("ReadDotLines2: %v, %v", s, err) } } @@ -110,13 +111,13 @@ func TestReadDotBytes(t *testing.T) { r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n") b, err := r.ReadDotBytes() want := []byte("dotlines\nfoo\n.bar\n..baz\nquux\n\n") - if !reflect.DeepEqual(b, want) || err != nil { + if !slices.Equal(b, want) || err != nil { t.Fatalf("ReadDotBytes: %q, %v", b, err) } b, err = r.ReadDotBytes() want = []byte("anot.her\n") - if !reflect.DeepEqual(b, want) || err != io.ErrUnexpectedEOF { + if !slices.Equal(b, want) || err != io.ErrUnexpectedEOF { t.Fatalf("ReadDotBytes2: %q, %v", b, err) } } diff --git a/src/net/udpsock.go b/src/net/udpsock.go index 4f8acb7f..56aabffa 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -27,7 +27,7 @@ type UDPAddr struct { Zone string // IPv6 scoped addressing zone } -// AddrPort returns the UDPAddr a as a netip.AddrPort. +// AddrPort returns the [UDPAddr] a as a [netip.AddrPort]. // // If a.Port does not fit in a uint16, it's silently truncated. // @@ -82,7 +82,7 @@ func (a *UDPAddr) opAddr() Addr { // recommended, because it will return at most one of the host name's // IP addresses. // -// See func Dial for a description of the network and address +// See func [Dial] for a description of the network and address // parameters. func ResolveUDPAddr(network, address string) (*UDPAddr, error) { switch network { @@ -99,7 +99,7 @@ func ResolveUDPAddr(network, address string) (*UDPAddr, error) { return addrs.forResolve(network, address).(*UDPAddr), nil } -// UDPAddrFromAddrPort returns addr as a UDPAddr. If addr.IsValid() is false, +// UDPAddrFromAddrPort returns addr as a [UDPAddr]. If addr.IsValid() is false, // then the returned UDPAddr will contain a nil IP field, indicating an // address family-agnostic unspecified address. func UDPAddrFromAddrPort(addr netip.AddrPort) *UDPAddr { @@ -117,14 +117,14 @@ type addrPortUDPAddr struct { func (addrPortUDPAddr) Network() string { return "udp" } -// UDPConn is the implementation of the Conn and PacketConn interfaces +// UDPConn is the implementation of the [Conn] and [PacketConn] interfaces // for UDP network connections. type UDPConn struct { conn } // SyscallConn returns a raw network connection. -// This implements the syscall.Conn interface. +// This implements the [syscall.Conn] interface. func (c *UDPConn) SyscallConn() (syscall.RawConn, error) { if !c.ok() { return nil, syscall.EINVAL @@ -132,7 +132,7 @@ func (c *UDPConn) SyscallConn() (syscall.RawConn, error) { return newRawConn(c.fd), nil } -// ReadFromUDP acts like ReadFrom but returns a UDPAddr. +// ReadFromUDP acts like [UDPConn.ReadFrom] but returns a UDPAddr. func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { // This function is designed to allow the caller to control the lifetime // of the returned *UDPAddr and thereby prevent an allocation. @@ -153,7 +153,7 @@ func (c *UDPConn) readFromUDP(b []byte, addr *UDPAddr) (int, *UDPAddr, error) { return n, addr, err } -// ReadFrom implements the PacketConn ReadFrom method. +// ReadFrom implements the [PacketConn] ReadFrom method. func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { n, addr, err := c.readFromUDP(b, &UDPAddr{}) if addr == nil { @@ -163,11 +163,11 @@ func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { return n, addr, err } -// ReadFromUDPAddrPort acts like ReadFrom but returns a netip.AddrPort. +// ReadFromUDPAddrPort acts like ReadFrom but returns a [netip.AddrPort]. // // If c is bound to an unspecified address, the returned // netip.AddrPort's address might be an IPv4-mapped IPv6 address. -// Use netip.Addr.Unmap to get the address without the IPv6 prefix. +// Use [netip.Addr.Unmap] to get the address without the IPv6 prefix. func (c *UDPConn) ReadFromUDPAddrPort(b []byte) (n int, addr netip.AddrPort, err error) { if !c.ok() { return 0, netip.AddrPort{}, syscall.EINVAL @@ -184,7 +184,7 @@ func (c *UDPConn) ReadFromUDPAddrPort(b []byte) (n int, addr netip.AddrPort, err // bytes copied into b, the number of bytes copied into oob, the flags // that were set on the message and the source address of the message. // -// The packages golang.org/x/net/ipv4 and golang.org/x/net/ipv6 can be +// The packages [golang.org/x/net/ipv4] and [golang.org/x/net/ipv6] can be // used to manipulate IP-level socket options in oob. func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { var ap netip.AddrPort @@ -195,7 +195,7 @@ func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, return } -// ReadMsgUDPAddrPort is like ReadMsgUDP but returns an netip.AddrPort instead of a UDPAddr. +// ReadMsgUDPAddrPort is like [UDPConn.ReadMsgUDP] but returns an [netip.AddrPort] instead of a [UDPAddr]. func (c *UDPConn) ReadMsgUDPAddrPort(b, oob []byte) (n, oobn, flags int, addr netip.AddrPort, err error) { if !c.ok() { return 0, 0, 0, netip.AddrPort{}, syscall.EINVAL @@ -207,7 +207,7 @@ func (c *UDPConn) ReadMsgUDPAddrPort(b, oob []byte) (n, oobn, flags int, addr ne return } -// WriteToUDP acts like WriteTo but takes a UDPAddr. +// WriteToUDP acts like [UDPConn.WriteTo] but takes a [UDPAddr]. func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { if !c.ok() { return 0, syscall.EINVAL @@ -219,7 +219,7 @@ func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { return n, err } -// WriteToUDPAddrPort acts like WriteTo but takes a netip.AddrPort. +// WriteToUDPAddrPort acts like [UDPConn.WriteTo] but takes a [netip.AddrPort]. func (c *UDPConn) WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (int, error) { if !c.ok() { return 0, syscall.EINVAL @@ -231,7 +231,7 @@ func (c *UDPConn) WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (int, error) return n, err } -// WriteTo implements the PacketConn WriteTo method. +// WriteTo implements the [PacketConn] WriteTo method. func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { if !c.ok() { return 0, syscall.EINVAL @@ -253,7 +253,7 @@ func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { // data is copied from oob. It returns the number of payload and // out-of-band bytes written. // -// The packages golang.org/x/net/ipv4 and golang.org/x/net/ipv6 can be +// The packages [golang.org/x/net/ipv4] and [golang.org/x/net/ipv6] can be // used to manipulate IP-level socket options in oob. func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { if !c.ok() { @@ -266,7 +266,7 @@ func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err er return } -// WriteMsgUDPAddrPort is like WriteMsgUDP but takes a netip.AddrPort instead of a UDPAddr. +// WriteMsgUDPAddrPort is like [UDPConn.WriteMsgUDP] but takes a [netip.AddrPort] instead of a [UDPAddr]. func (c *UDPConn) WriteMsgUDPAddrPort(b, oob []byte, addr netip.AddrPort) (n, oobn int, err error) { if !c.ok() { return 0, 0, syscall.EINVAL @@ -280,9 +280,9 @@ func (c *UDPConn) WriteMsgUDPAddrPort(b, oob []byte, addr netip.AddrPort) (n, oo func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } -// DialUDP acts like Dial for UDP networks. +// DialUDP acts like [Dial] for UDP networks. // -// The network must be a UDP network name; see func Dial for details. +// The network must be a UDP network name; see func [Dial] for details. // // If laddr is nil, a local address is automatically chosen. // If the IP field of raddr is nil or an unspecified IP address, the @@ -304,9 +304,9 @@ func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { return c, nil } -// ListenUDP acts like ListenPacket for UDP networks. +// ListenUDP acts like [ListenPacket] for UDP networks. // -// The network must be a UDP network name; see func Dial for details. +// The network must be a UDP network name; see func [Dial] for details. // // If the IP field of laddr is nil or an unspecified IP address, // ListenUDP listens on all available IP addresses of the local system @@ -330,10 +330,10 @@ func ListenUDP(network string, laddr *UDPAddr) (*UDPConn, error) { return c, nil } -// ListenMulticastUDP acts like ListenPacket for UDP networks but +// ListenMulticastUDP acts like [ListenPacket] for UDP networks but // takes a group address on a specific network interface. // -// The network must be a UDP network name; see func Dial for details. +// The network must be a UDP network name; see func [Dial] for details. // // ListenMulticastUDP listens on all available IP addresses of the // local system including the group, multicast IP address. @@ -345,8 +345,8 @@ func ListenUDP(network string, laddr *UDPAddr) (*UDPConn, error) { // chosen. // // ListenMulticastUDP is just for convenience of simple, small -// applications. There are golang.org/x/net/ipv4 and -// golang.org/x/net/ipv6 packages for general purpose uses. +// applications. There are [golang.org/x/net/ipv4] and +// [golang.org/x/net/ipv6] packages for general purpose uses. // // Note that ListenMulticastUDP will set the IP_MULTICAST_LOOP socket option // to 0 under IPPROTO_IP, to disable loopback of multicast packets. diff --git a/src/net/udpsock_test.go b/src/net/udpsock_test.go index 8a21aa73..6dacc81d 100644 --- a/src/net/udpsock_test.go +++ b/src/net/udpsock_test.go @@ -7,6 +7,7 @@ package net import ( "errors" "fmt" + "internal/asan" "internal/testenv" "net/netip" "os" @@ -340,7 +341,7 @@ func TestUDPZeroBytePayload(t *testing.T) { switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) - case "darwin", "ios": + case "ios": testenv.SkipFlaky(t, 29225) } if !testableNetwork("udp") { @@ -493,6 +494,9 @@ func TestAllocs(t *testing.T) { if !testableNetwork("udp4") { t.Skipf("skipping: udp4 not available") } + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } // Optimizations are required to remove the allocs. testenv.SkipIfOptimizationOff(t) diff --git a/src/net/url/example_test.go b/src/net/url/example_test.go index a1913508..311ba5c3 100644 --- a/src/net/url/example_test.go +++ b/src/net/url/example_test.go @@ -253,6 +253,18 @@ func ExampleURL_IsAbs() { // true } +func ExampleURL_JoinPath() { + u, err := url.Parse("https://example.com/foo/bar") + if err != nil { + log.Fatal(err) + } + + fmt.Println(u.JoinPath("baz", "qux")) + + // Output: + // https://example.com/foo/bar/baz/qux +} + func ExampleURL_MarshalBinary() { u, _ := url.Parse("https://example.org") b, err := u.MarshalBinary() diff --git a/src/net/url/url.go b/src/net/url/url.go index 7beaef1b..8a8de1c6 100644 --- a/src/net/url/url.go +++ b/src/net/url/url.go @@ -13,6 +13,7 @@ package url import ( "errors" "fmt" + "maps" "path" "slices" "strconv" @@ -1004,12 +1005,7 @@ func (v Values) Encode() string { return "" } var buf strings.Builder - keys := make([]string, 0, len(v)) - for k := range v { - keys = append(keys, k) - } - slices.Sort(keys) - for _, k := range keys { + for _, k := range slices.Sorted(maps.Keys(v)) { vs := v[k] keyEscaped := QueryEscape(k) for _, v := range vs { @@ -1219,7 +1215,11 @@ func splitHostPort(hostPort string) (host, port string) { // Would like to implement MarshalText/UnmarshalText but that will change the JSON representation of URLs. func (u *URL) MarshalBinary() (text []byte, err error) { - return []byte(u.String()), nil + return u.AppendBinary(nil) +} + +func (u *URL) AppendBinary(b []byte) ([]byte, error) { + return append(b, u.String()...), nil } func (u *URL) UnmarshalBinary(text []byte) error { diff --git a/src/net/url/url_test.go b/src/net/url/url_test.go index 68219c3d..16e08b63 100644 --- a/src/net/url/url_test.go +++ b/src/net/url/url_test.go @@ -1866,6 +1866,7 @@ func TestURLHostnameAndPort(t *testing.T) { var _ encodingPkg.BinaryMarshaler = (*URL)(nil) var _ encodingPkg.BinaryUnmarshaler = (*URL)(nil) +var _ encodingPkg.BinaryAppender = (*URL)(nil) func TestJSON(t *testing.T) { u, err := Parse("https://www.google.com/x?y=z") diff --git a/src/os/copy_test.go b/src/os/copy_test.go index 82346ca4..6fe7f6e5 100644 --- a/src/os/copy_test.go +++ b/src/os/copy_test.go @@ -7,6 +7,7 @@ package os_test import ( "bytes" "errors" + "fmt" "io" "math/rand/v2" "net" @@ -70,28 +71,135 @@ func TestLargeCopyViaNetwork(t *testing.T) { } } +func TestCopyFileToFile(t *testing.T) { + const size = 1 * 1024 * 1024 + dir := t.TempDir() + + src, err := os.Create(dir + "/src") + if err != nil { + t.Fatal(err) + } + defer src.Close() + if _, err := io.CopyN(src, newRandReader(), size); err != nil { + t.Fatal(err) + } + if _, err := src.Seek(0, 0); err != nil { + t.Fatal(err) + } + + mustSeek := func(f *os.File, offset int64, whence int) int64 { + ret, err := f.Seek(offset, whence) + if err != nil { + t.Fatal(err) + } + return ret + } + + for _, srcStart := range []int64{0, 100, size} { + remaining := size - srcStart + for _, dstStart := range []int64{0, 200} { + for _, limit := range []int64{remaining, remaining - 100, size * 2, 0} { + if limit < 0 { + continue + } + name := fmt.Sprintf("srcStart=%v/dstStart=%v/limit=%v", srcStart, dstStart, limit) + t.Run(name, func(t *testing.T) { + dst, err := os.CreateTemp(dir, "dst") + if err != nil { + t.Fatal(err) + } + defer dst.Close() + defer os.Remove(dst.Name()) + + mustSeek(src, srcStart, io.SeekStart) + if _, err := io.CopyN(dst, zeroReader{}, dstStart); err != nil { + t.Fatal(err) + } + + var copied int64 + if limit == 0 { + copied, err = io.Copy(dst, src) + } else { + copied, err = io.CopyN(dst, src, limit) + } + if limit > remaining { + if err != io.EOF { + t.Errorf("Copy: %v; want io.EOF", err) + } + } else { + if err != nil { + t.Errorf("Copy: %v; want nil", err) + } + } + + wantCopied := remaining + if limit != 0 { + wantCopied = min(limit, wantCopied) + } + if copied != wantCopied { + t.Errorf("copied %v bytes, want %v", copied, wantCopied) + } + + srcPos := mustSeek(src, 0, io.SeekCurrent) + wantSrcPos := srcStart + wantCopied + if srcPos != wantSrcPos { + t.Errorf("source position = %v, want %v", srcPos, wantSrcPos) + } + + dstPos := mustSeek(dst, 0, io.SeekCurrent) + wantDstPos := dstStart + wantCopied + if dstPos != wantDstPos { + t.Errorf("destination position = %v, want %v", dstPos, wantDstPos) + } + + mustSeek(dst, 0, io.SeekStart) + rr := newRandReader() + io.CopyN(io.Discard, rr, srcStart) + wantReader := io.MultiReader( + io.LimitReader(zeroReader{}, dstStart), + io.LimitReader(rr, wantCopied), + ) + if err := compareReaders(dst, wantReader); err != nil { + t.Fatal(err) + } + }) + + } + } + } +} + func compareReaders(a, b io.Reader) error { bufa := make([]byte, 4096) bufb := make([]byte, 4096) + off := 0 for { na, erra := io.ReadFull(a, bufa) - if erra != nil && erra != io.EOF { + if erra != nil && erra != io.EOF && erra != io.ErrUnexpectedEOF { return erra } nb, errb := io.ReadFull(b, bufb) - if errb != nil && errb != io.EOF { + if errb != nil && errb != io.EOF && errb != io.ErrUnexpectedEOF { return errb } if !bytes.Equal(bufa[:na], bufb[:nb]) { return errors.New("contents mismatch") } - if erra == io.EOF && errb == io.EOF { + if erra != nil && errb != nil { break } + off += len(bufa) } return nil } +type zeroReader struct{} + +func (r zeroReader) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + type randReader struct { rand *rand.Rand } @@ -101,16 +209,8 @@ func newRandReader() *randReader { } func (r *randReader) Read(p []byte) (int, error) { - var v uint64 - var n int for i := range p { - if n == 0 { - v = r.rand.Uint64() - n = 8 - } - p[i] = byte(v & 0xff) - v >>= 8 - n-- + p[i] = byte(r.rand.Uint32() & 0xff) } return len(p), nil } diff --git a/src/os/dir.go b/src/os/dir.go index 04392193..939b208d 100644 --- a/src/os/dir.go +++ b/src/os/dir.go @@ -145,6 +145,9 @@ func ReadDir(name string) ([]DirEntry, error) { // // Symbolic links in dir are followed. // +// New files added to fsys (including if dir is a subdirectory of fsys) +// while CopyFS is running are not guaranteed to be copied. +// // Copying stops at and returns the first error encountered. func CopyFS(dir string, fsys fs.FS) error { return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { diff --git a/src/os/dir_unix.go b/src/os/dir_unix.go index d8b4faa0..eadc1660 100644 --- a/src/os/dir_unix.go +++ b/src/os/dir_unix.go @@ -175,11 +175,11 @@ func readIntBE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.BeUint16(b)) + return uint64(byteorder.BEUint16(b)) case 4: - return uint64(byteorder.BeUint32(b)) + return uint64(byteorder.BEUint32(b)) case 8: - return uint64(byteorder.BeUint64(b)) + return uint64(byteorder.BEUint64(b)) default: panic("syscall: readInt with unsupported size") } @@ -190,11 +190,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.LeUint16(b)) + return uint64(byteorder.LEUint16(b)) case 4: - return uint64(byteorder.LeUint32(b)) + return uint64(byteorder.LEUint32(b)) case 8: - return uint64(byteorder.LeUint64(b)) + return uint64(byteorder.LEUint64(b)) default: panic("syscall: readInt with unsupported size") } diff --git a/src/os/eloop_netbsd.go b/src/os/eloop_netbsd.go new file mode 100644 index 00000000..670c88a8 --- /dev/null +++ b/src/os/eloop_netbsd.go @@ -0,0 +1,19 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build netbsd + +package os + +import "syscall" + +// isNoFollowErr reports whether err may result from O_NOFOLLOW blocking an open operation. +func isNoFollowErr(err error) bool { + // NetBSD returns EFTYPE, but check the other possibilities as well. + switch err { + case syscall.ELOOP, syscall.EMLINK, syscall.EFTYPE: + return true + } + return false +} diff --git a/src/os/eloop_other.go b/src/os/eloop_other.go new file mode 100644 index 00000000..aace57bd --- /dev/null +++ b/src/os/eloop_other.go @@ -0,0 +1,27 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || openbsd || solaris || wasip1 + +package os + +import ( + "runtime" + "syscall" +) + +// isNoFollowErr reports whether err may result from O_NOFOLLOW blocking an open operation. +func isNoFollowErr(err error) bool { + switch err { + case syscall.ELOOP, syscall.EMLINK: + return true + } + if runtime.GOOS == "dragonfly" { + // Dragonfly appears to return EINVAL from openat in this case. + if err == syscall.EINVAL { + return true + } + } + return false +} diff --git a/src/os/env_test.go b/src/os/env_test.go index 5809f4b8..e3de6419 100644 --- a/src/os/env_test.go +++ b/src/os/env_test.go @@ -6,7 +6,7 @@ package os_test import ( . "os" - "reflect" + "slices" "strings" "testing" ) @@ -91,7 +91,7 @@ func TestConsistentEnviron(t *testing.T) { e0 := Environ() for i := 0; i < 10; i++ { e1 := Environ() - if !reflect.DeepEqual(e0, e1) { + if !slices.Equal(e0, e1) { t.Fatalf("environment changed") } } diff --git a/src/os/error_errno.go b/src/os/error_errno.go index c8140461..66225833 100644 --- a/src/os/error_errno.go +++ b/src/os/error_errno.go @@ -9,3 +9,9 @@ package os import "syscall" type syscallErrorType = syscall.Errno + +const ( + errENOSYS = syscall.ENOSYS + errERANGE = syscall.ERANGE + errENOMEM = syscall.ENOMEM +) diff --git a/src/os/error_plan9.go b/src/os/error_plan9.go index af6065db..35026554 100644 --- a/src/os/error_plan9.go +++ b/src/os/error_plan9.go @@ -7,3 +7,7 @@ package os import "syscall" type syscallErrorType = syscall.ErrorString + +var errENOSYS = syscall.NewError("function not implemented") +var errERANGE = syscall.NewError("out of range") +var errENOMEM = syscall.NewError("cannot allocate memory") diff --git a/src/os/exec/dot_test.go b/src/os/exec/dot_test.go index ed4bad23..1bf0d9c7 100644 --- a/src/os/exec/dot_test.go +++ b/src/os/exec/dot_test.go @@ -38,8 +38,7 @@ func TestLookPath(t *testing.T) { if err := os.WriteFile(filepath.Join(tmpDir, executable), []byte{1, 2, 3}, 0777); err != nil { t.Fatal(err) } - chdir(t, tmpDir) - t.Setenv("PWD", tmpDir) + t.Chdir(tmpDir) t.Logf(". is %#q", tmpDir) origPath := os.Getenv(pathVar) diff --git a/src/os/exec/env_test.go b/src/os/exec/env_test.go index ea06af38..9fd022b2 100644 --- a/src/os/exec/env_test.go +++ b/src/os/exec/env_test.go @@ -5,7 +5,7 @@ package exec import ( - "reflect" + "slices" "testing" ) @@ -60,7 +60,7 @@ func TestDedupEnv(t *testing.T) { } for _, tt := range tests { got, err := dedupEnvCase(tt.noCase, tt.nulOK, tt.in) - if !reflect.DeepEqual(got, tt.want) || (err != nil) != tt.wantErr { + if !slices.Equal(got, tt.want) || (err != nil) != tt.wantErr { t.Errorf("Dedup(%v, %q) = %q, %v; want %q, error:%v", tt.noCase, tt.in, got, err, tt.want, tt.wantErr) } } diff --git a/src/os/exec/exec.go b/src/os/exec/exec.go index da9f68fe..fecfc97d 100644 --- a/src/os/exec/exec.go +++ b/src/os/exec/exec.go @@ -166,11 +166,27 @@ type Cmd struct { // value in the slice for each duplicate key is used. // As a special case on Windows, SYSTEMROOT is always added if // missing and not explicitly set to the empty string. + // + // See also the Dir field, which may set PWD in the environment. Env []string // Dir specifies the working directory of the command. // If Dir is the empty string, Run runs the command in the // calling process's current directory. + // + // On Unix systems, the value of Dir also determines the + // child process's PWD environment variable if not otherwise + // specified. A Unix process represents its working directory + // not by name but as an implicit reference to a node in the + // file tree. So, if the child process obtains its working + // directory by calling a function such as C's getcwd, which + // computes the canonical name by walking up the file tree, it + // will not recover the original value of Dir if that value + // was an alias involving symbolic links. However, if the + // child process calls Go's [os.Getwd] or GNU C's + // get_current_dir_name, and the value of PWD is an alias for + // the current directory, those functions will return the + // value of PWD, which matches the value of Dir. Dir string // Stdin specifies the process's standard input. @@ -984,7 +1000,9 @@ func (c *Cmd) awaitGoroutines(timer *time.Timer) error { // Output runs the command and returns its standard output. // Any returned error will usually be of type [*ExitError]. -// If c.Stderr was nil, Output populates [ExitError.Stderr]. +// If c.Stderr was nil and the returned error is of type +// [*ExitError], Output populates the Stderr field of the +// returned error. func (c *Cmd) Output() ([]byte, error) { if c.Stdout != nil { return nil, errors.New("exec: Stdout already set") diff --git a/src/os/exec/exec_posix_test.go b/src/os/exec/exec_posix_test.go index 5d828b34..45604203 100644 --- a/src/os/exec/exec_posix_test.go +++ b/src/os/exec/exec_posix_test.go @@ -13,8 +13,8 @@ import ( "os" "os/user" "path/filepath" - "reflect" "runtime" + "slices" "strconv" "strings" "syscall" @@ -184,7 +184,7 @@ func TestImplicitPWD(t *testing.T) { wantPWDs = nil } } - if !reflect.DeepEqual(pwds, wantPWDs) { + if !slices.Equal(pwds, wantPWDs) { t.Errorf("PWD entries in cmd.Environ():\n\t%s\nwant:\n\t%s", strings.Join(pwds, "\n\t"), strings.Join(wantPWDs, "\n\t")) } @@ -257,7 +257,7 @@ func TestExplicitPWD(t *testing.T) { } wantPWDs := []string{tc.pwd} - if !reflect.DeepEqual(pwds, wantPWDs) { + if !slices.Equal(pwds, wantPWDs) { t.Errorf("PWD entries in cmd.Environ():\n\t%s\nwant:\n\t%s", strings.Join(pwds, "\n\t"), strings.Join(wantPWDs, "\n\t")) } diff --git a/src/os/exec/exec_test.go b/src/os/exec/exec_test.go index a0bb89e2..8c623871 100644 --- a/src/os/exec/exec_test.go +++ b/src/os/exec/exec_test.go @@ -159,64 +159,16 @@ func helperCommandContext(t *testing.T, ctx context.Context, name string, args . helperCommandUsed.LoadOrStore(name, true) t.Helper() - testenv.MustHaveExec(t) - + exe := testenv.Executable(t) cs := append([]string{name}, args...) if ctx != nil { - cmd = exec.CommandContext(ctx, exePath(t), cs...) + cmd = exec.CommandContext(ctx, exe, cs...) } else { - cmd = exec.Command(exePath(t), cs...) + cmd = exec.Command(exe, cs...) } return cmd } -// exePath returns the path to the running executable. -func exePath(t testing.TB) string { - exeOnce.Do(func() { - // Use os.Executable instead of os.Args[0] in case the caller modifies - // cmd.Dir: if the test binary is invoked like "./exec.test", it should - // not fail spuriously. - exeOnce.path, exeOnce.err = os.Executable() - }) - - if exeOnce.err != nil { - if t == nil { - panic(exeOnce.err) - } - t.Fatal(exeOnce.err) - } - - return exeOnce.path -} - -var exeOnce struct { - path string - err error - sync.Once -} - -func chdir(t *testing.T, dir string) { - t.Helper() - - prev, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - if err := os.Chdir(dir); err != nil { - t.Fatal(err) - } - t.Logf("Chdir(%#q)", dir) - - t.Cleanup(func() { - if err := os.Chdir(prev); err != nil { - // Couldn't chdir back to the original working directory. - // panic instead of t.Fatal so that we don't run other tests - // in an unexpected location. - panic("couldn't restore working directory: " + err.Error()) - } - }) -} - var helperCommandUsed sync.Map var helperCommands = map[string]func(...string){ @@ -1201,7 +1153,7 @@ func cmdHang(args ...string) { pid := os.Getpid() if *subsleep != 0 { - cmd := exec.Command(exePath(nil), "hang", subsleep.String(), "-read=true", "-probe="+probe.String()) + cmd := exec.Command(testenv.Executable(nil), "hang", subsleep.String(), "-read=true", "-probe="+probe.String()) cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr out, err := cmd.StdoutPipe() diff --git a/src/os/exec/lp_unix_test.go b/src/os/exec/lp_unix_test.go index 1503ddae..ea7ec11c 100644 --- a/src/os/exec/lp_unix_test.go +++ b/src/os/exec/lp_unix_test.go @@ -16,7 +16,7 @@ func TestLookPathUnixEmptyPath(t *testing.T) { // Not parallel: uses Chdir and Setenv. tmp := t.TempDir() - chdir(t, tmp) + t.Chdir(tmp) f, err := os.OpenFile("exec_me", os.O_CREATE|os.O_EXCL, 0700) if err != nil { diff --git a/src/os/exec/lp_windows.go b/src/os/exec/lp_windows.go index 0e058d41..12256743 100644 --- a/src/os/exec/lp_windows.go +++ b/src/os/exec/lp_windows.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "strings" - "syscall" ) // ErrNotFound is the error resulting if a path search failed to find an executable file. @@ -154,7 +153,7 @@ func lookPath(file string, exts []string) (string, error) { dotf string dotErr error ) - if _, found := syscall.Getenv("NoDefaultCurrentDirectoryInExePath"); !found { + if _, found := os.LookupEnv("NoDefaultCurrentDirectoryInExePath"); !found { if f, err := findExecutable(filepath.Join(".", file), exts); err == nil { if execerrdot.Value() == "0" { execerrdot.IncNonDefault() diff --git a/src/os/exec/lp_windows_test.go b/src/os/exec/lp_windows_test.go index a92a2979..01eda04c 100644 --- a/src/os/exec/lp_windows_test.go +++ b/src/os/exec/lp_windows_test.go @@ -25,13 +25,8 @@ func init() { registerHelperCommand("printpath", cmdPrintPath) } -func cmdPrintPath(args ...string) { - exe, err := os.Executable() - if err != nil { - fmt.Fprintf(os.Stderr, "Executable: %v\n", err) - os.Exit(1) - } - fmt.Println(exe) +func cmdPrintPath(_ ...string) { + fmt.Println(testenv.Executable(nil)) } // makePATH returns a PATH variable referring to the @@ -82,7 +77,7 @@ func installProgs(t *testing.T, root string, files []string) { // (We use a copy instead of just a symlink to ensure that os.Executable // always reports an unambiguous path, regardless of how it is implemented.) func installExe(t *testing.T, dstPath string) { - src, err := os.Open(exePath(t)) + src, err := os.Open(testenv.Executable(t)) if err != nil { t.Fatal(err) } @@ -319,7 +314,7 @@ func TestLookPathWindows(t *testing.T) { t.Setenv("PATH", pathVar) t.Logf("set PATH=%s", pathVar) - chdir(t, root) + t.Chdir(root) if !testing.Short() && !(tt.skipCmdExeCheck || errors.Is(tt.wantErr, exec.ErrDot)) { // Check that cmd.exe, which is our source of ground truth, @@ -554,7 +549,7 @@ func TestCommand(t *testing.T) { t.Setenv("PATH", pathVar) t.Logf("set PATH=%s", pathVar) - chdir(t, root) + t.Chdir(root) cmd := exec.Command(tt.arg0, "printpath") cmd.Dir = filepath.Join(root, tt.dir) diff --git a/src/os/exec_unix.go b/src/os/exec_unix.go index ba6146ad..34467ac7 100644 --- a/src/os/exec_unix.go +++ b/src/os/exec_unix.go @@ -63,17 +63,12 @@ func (p *Process) pidWait() (*ProcessState, error) { var ( status syscall.WaitStatus rusage syscall.Rusage - pid1 int - e error ) - for { - pid1, e = syscall.Wait4(p.Pid, &status, 0, &rusage) - if e != syscall.EINTR { - break - } - } - if e != nil { - return nil, NewSyscallError("wait", e) + pid1, err := ignoringEINTR2(func() (int, error) { + return syscall.Wait4(p.Pid, &status, 0, &rusage) + }) + if err != nil { + return nil, NewSyscallError("wait", err) } p.pidDeactivate(statusDone) return &ProcessState{ diff --git a/src/os/exec_windows.go b/src/os/exec_windows.go index a2d7d343..f5e3a4ff 100644 --- a/src/os/exec_windows.go +++ b/src/os/exec_windows.go @@ -54,7 +54,7 @@ func (p *Process) wait() (ps *ProcessState, err error) { if !isWin10AndAbove { defer time.Sleep(5 * time.Millisecond) } - + defer p.Release() return &ProcessState{p.Pid, syscall.WaitStatus{ExitCode: ec}, &u}, nil } diff --git a/src/os/executable_dragonfly.go b/src/os/executable_dragonfly.go index 19c2ae89..939c6f6e 100644 --- a/src/os/executable_dragonfly.go +++ b/src/os/executable_dragonfly.go @@ -10,3 +10,5 @@ const ( _KERN_PROC = 14 _KERN_PROC_PATHNAME = 9 ) + +var executableMIB = [4]int32{_CTL_KERN, _KERN_PROC, _KERN_PROC_PATHNAME, -1} diff --git a/src/os/executable_freebsd.go b/src/os/executable_freebsd.go index 95f1a93c..da40fcb3 100644 --- a/src/os/executable_freebsd.go +++ b/src/os/executable_freebsd.go @@ -10,3 +10,5 @@ const ( _KERN_PROC = 14 _KERN_PROC_PATHNAME = 12 ) + +var executableMIB = [4]int32{_CTL_KERN, _KERN_PROC, _KERN_PROC_PATHNAME, -1} diff --git a/src/os/executable_netbsd.go b/src/os/executable_netbsd.go new file mode 100644 index 00000000..fd075390 --- /dev/null +++ b/src/os/executable_netbsd.go @@ -0,0 +1,14 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +// From NetBSD's +const ( + _CTL_KERN = 1 + _KERN_PROC_ARGS = 48 + _KERN_PROC_PATHNAME = 5 +) + +var executableMIB = [4]int32{_CTL_KERN, _KERN_PROC_ARGS, -1, _KERN_PROC_PATHNAME} diff --git a/src/os/executable_procfs.go b/src/os/executable_procfs.go index 6a2cd10b..a52631c0 100644 --- a/src/os/executable_procfs.go +++ b/src/os/executable_procfs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux || netbsd +//go:build linux package os @@ -19,8 +19,6 @@ func executable() (string, error) { return "", errors.New("Executable not implemented for " + runtime.GOOS) case "linux", "android": procfn = "/proc/self/exe" - case "netbsd": - procfn = "/proc/curproc/exe" } path, err := Readlink(procfn) diff --git a/src/os/executable_sysctl.go b/src/os/executable_sysctl.go index 3c2aeacf..8b52e92c 100644 --- a/src/os/executable_sysctl.go +++ b/src/os/executable_sysctl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build freebsd || dragonfly +//go:build freebsd || dragonfly || netbsd package os @@ -12,11 +12,9 @@ import ( ) func executable() (string, error) { - mib := [4]int32{_CTL_KERN, _KERN_PROC, _KERN_PROC_PATHNAME, -1} - n := uintptr(0) // get length - _, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0) + _, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&executableMIB[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0) if err != 0 { return "", err } @@ -24,7 +22,7 @@ func executable() (string, error) { return "", nil } buf := make([]byte, n) - _, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0) + _, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&executableMIB[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0) if err != 0 { return "", err } diff --git a/src/os/executable_test.go b/src/os/executable_test.go index 98b72d7d..1770843c 100644 --- a/src/os/executable_test.go +++ b/src/os/executable_test.go @@ -13,16 +13,30 @@ import ( "testing" ) -const executable_EnvVar = "OSTEST_OUTPUT_EXECPATH" - func TestExecutable(t *testing.T) { - testenv.MustHaveExec(t) - t.Parallel() + const helperEnvVar = "OSTEST_OUTPUT_EXECPATH" - ep, err := os.Executable() - if err != nil { - t.Fatalf("Executable failed: %v", err) + if os.Getenv(helperEnvVar) != "" { + // First chdir to another path. + dir := "/" + if runtime.GOOS == "windows" { + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + dir = filepath.VolumeName(cwd) + } + os.Chdir(dir) + if ep, err := os.Executable(); err != nil { + fmt.Fprint(os.Stderr, "ERROR: ", err) + } else { + fmt.Fprint(os.Stderr, ep) + } + os.Exit(0) } + + t.Parallel() + ep := testenv.Executable(t) // we want fn to be of the form "dir/prog" dir := filepath.Dir(filepath.Dir(ep)) fn, err := filepath.Rel(dir, ep) @@ -30,7 +44,7 @@ func TestExecutable(t *testing.T) { t.Fatalf("filepath.Rel: %v", err) } - cmd := testenv.Command(t, fn, "-test.run=^$") + cmd := testenv.Command(t, fn, "-test.run=^"+t.Name()+"$") // make child start with a relative program path cmd.Dir = dir cmd.Path = fn @@ -41,7 +55,7 @@ func TestExecutable(t *testing.T) { // get real path of the executable without influenced by argv[0]. cmd.Args[0] = "-" } - cmd.Env = append(cmd.Environ(), fmt.Sprintf("%s=1", executable_EnvVar)) + cmd.Env = append(cmd.Environ(), fmt.Sprintf("%s=1", helperEnvVar)) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("exec(self) failed: %v", err) @@ -67,27 +81,6 @@ func sameFile(fn1, fn2 string) bool { return os.SameFile(fi1, fi2) } -func init() { - if e := os.Getenv(executable_EnvVar); e != "" { - // first chdir to another path - dir := "/" - if runtime.GOOS == "windows" { - cwd, err := os.Getwd() - if err != nil { - panic(err) - } - dir = filepath.VolumeName(cwd) - } - os.Chdir(dir) - if ep, err := os.Executable(); err != nil { - fmt.Fprint(os.Stderr, "ERROR: ", err) - } else { - fmt.Fprint(os.Stderr, ep) - } - os.Exit(0) - } -} - func TestExecutableDeleted(t *testing.T) { testenv.MustHaveGoBuild(t) switch runtime.GOOS { diff --git a/src/os/export_freebsd_test.go b/src/os/export_freebsd_test.go new file mode 100644 index 00000000..56bfcc6c --- /dev/null +++ b/src/os/export_freebsd_test.go @@ -0,0 +1,9 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +var ( + PollCopyFileRangeP = &pollCopyFileRange +) diff --git a/src/os/fifo_test.go b/src/os/fifo_test.go index 3b7e5eac..111dd2aa 100644 --- a/src/os/fifo_test.go +++ b/src/os/fifo_test.go @@ -161,10 +161,7 @@ func TestNonPollable(t *testing.T) { // Issue 60211. func TestOpenFileNonBlocking(t *testing.T) { - exe, err := os.Executable() - if err != nil { - t.Skipf("can't find executable: %v", err) - } + exe := testenv.Executable(t) f, err := os.OpenFile(exe, os.O_RDONLY|syscall.O_NONBLOCK, 0666) if err != nil { t.Fatal(err) diff --git a/src/os/file.go b/src/os/file.go index ad869fc4..a5063680 100644 --- a/src/os/file.go +++ b/src/os/file.go @@ -344,8 +344,13 @@ func Chdir(dir string) error { return &PathError{Op: "chdir", Path: dir, Err: e} } if runtime.GOOS == "windows" { + abs := filepathlite.IsAbs(dir) getwdCache.Lock() - getwdCache.dir = dir + if abs { + getwdCache.dir = dir + } else { + getwdCache.dir = "" + } getwdCache.Unlock() } if log := testlog.Logger(); log != nil { @@ -369,6 +374,7 @@ func Open(name string) (*File, error) { // it is truncated. If the file does not exist, it is created with mode 0o666 // (before umask). If successful, methods on the returned File can // be used for I/O; the associated file descriptor has mode O_RDWR. +// The directory containing the file must already exist. // If there is an error, it will be of type *PathError. func Create(name string) (*File, error) { return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666) @@ -377,7 +383,8 @@ func Create(name string) (*File, error) { // OpenFile is the generalized open call; most users will use Open // or Create instead. It opens the named file with specified flag // (O_RDONLY etc.). If the file does not exist, and the O_CREATE flag -// is passed, it is created with mode perm (before umask). If successful, +// is passed, it is created with mode perm (before umask); +// the containing directory must exist. If successful, // methods on the returned File can be used for I/O. // If there is an error, it will be of type *PathError. func OpenFile(name string, flag int, perm FileMode) (*File, error) { @@ -391,6 +398,8 @@ func OpenFile(name string, flag int, perm FileMode) (*File, error) { return f, nil } +var errPathEscapes = errors.New("path escapes from parent") + // openDir opens a file which is assumed to be a directory. As such, it skips // the syscalls that make the file descriptor non-blocking as these take time // and will fail on file descriptors for directories. @@ -404,6 +413,7 @@ var lstat = Lstat // Rename renames (moves) oldpath to newpath. // If newpath already exists and is not a directory, Rename replaces it. +// If newpath already exists and is a directory, Rename returns an error. // OS-specific restrictions may apply when oldpath and newpath are in different directories. // Even within the same directory, on non-Unix platforms Rename is not an atomic operation. // If there is an error, it will be of type *LinkError. @@ -472,8 +482,8 @@ func TempDir() string { // On Windows, it returns %LocalAppData%. // On Plan 9, it returns $home/lib/cache. // -// If the location cannot be determined (for example, $HOME is not defined), -// then it will return an error. +// If the location cannot be determined (for example, $HOME is not defined) or +// the path in $XDG_CACHE_HOME is relative, then it will return an error. func UserCacheDir() (string, error) { var dir string @@ -506,6 +516,8 @@ func UserCacheDir() (string, error) { return "", errors.New("neither $XDG_CACHE_HOME nor $HOME are defined") } dir += "/.cache" + } else if !filepathlite.IsAbs(dir) { + return "", errors.New("path in $XDG_CACHE_HOME is relative") } } @@ -523,8 +535,8 @@ func UserCacheDir() (string, error) { // On Windows, it returns %AppData%. // On Plan 9, it returns $home/lib. // -// If the location cannot be determined (for example, $HOME is not defined), -// then it will return an error. +// If the location cannot be determined (for example, $HOME is not defined) or +// the path in $XDG_CONFIG_HOME is relative, then it will return an error. func UserConfigDir() (string, error) { var dir string @@ -557,6 +569,8 @@ func UserConfigDir() (string, error) { return "", errors.New("neither $XDG_CONFIG_HOME nor $HOME are defined") } dir += "/.config" + } else if !filepathlite.IsAbs(dir) { + return "", errors.New("path in $XDG_CONFIG_HOME is relative") } } @@ -682,6 +696,8 @@ func (f *File) SyscallConn() (syscall.RawConn, error) { // a general substitute for a chroot-style security mechanism when the directory tree // contains arbitrary content. // +// Use [Root.FS] to obtain a fs.FS that prevents escapes from the tree via symbolic links. +// // The directory dir must not be "". // // The result implements [io/fs.StatFS], [io/fs.ReadFileFS] and @@ -786,7 +802,10 @@ func ReadFile(name string) ([]byte, error) { return nil, err } defer f.Close() + return readFileContents(f) +} +func readFileContents(f *File) ([]byte, error) { var size int if info, err := f.Stat(); err == nil { size64 := info.Size() diff --git a/src/os/file_plan9.go b/src/os/file_plan9.go index ef277dec..c123fe69 100644 --- a/src/os/file_plan9.go +++ b/src/os/file_plan9.go @@ -617,3 +617,7 @@ func newRawConn(file *File) (*rawConn, error) { func ignoringEINTR(fn func() error) error { return fn() } + +func ignoringEINTR2[T any](fn func() (T, error)) (T, error) { + return fn() +} diff --git a/src/os/file_posix.go b/src/os/file_posix.go index 8ff0ada4..f0cdfdae 100644 --- a/src/os/file_posix.go +++ b/src/os/file_posix.go @@ -254,3 +254,13 @@ func ignoringEINTR(fn func() error) error { } } } + +// ignoringEINTR2 is ignoringEINTR, but returning an additional value. +func ignoringEINTR2[T any](fn func() (T, error)) (T, error) { + for { + v, err := fn() + if err != syscall.EINTR { + return v, err + } + } +} diff --git a/src/os/file_unix.go b/src/os/file_unix.go index 37bfaa1a..b5c0baf3 100644 --- a/src/os/file_unix.go +++ b/src/os/file_unix.go @@ -306,7 +306,7 @@ func openDirNolog(name string) (*File, error) { e error ) ignoringEINTR(func() error { - r, s, e = open(name, O_RDONLY|syscall.O_CLOEXEC, 0) + r, s, e = open(name, O_RDONLY|syscall.O_CLOEXEC|syscall.O_DIRECTORY, 0) return e }) if e != nil { @@ -446,22 +446,15 @@ func Symlink(oldname, newname string) error { func readlink(name string) (string, error) { for len := 128; ; len *= 2 { b := make([]byte, len) - var ( - n int - e error - ) - for { - n, e = fixCount(syscall.Readlink(name, b)) - if e != syscall.EINTR { - break - } - } + n, err := ignoringEINTR2(func() (int, error) { + return fixCount(syscall.Readlink(name, b)) + }) // buffer too small - if (runtime.GOOS == "aix" || runtime.GOOS == "wasip1") && e == syscall.ERANGE { + if (runtime.GOOS == "aix" || runtime.GOOS == "wasip1") && err == syscall.ERANGE { continue } - if e != nil { - return "", &PathError{Op: "readlink", Path: name, Err: e} + if err != nil { + return "", &PathError{Op: "readlink", Path: name, Err: err} } if n < len { return string(b[0:n]), nil diff --git a/src/os/file_windows.go b/src/os/file_windows.go index cf652ca1..2160f1e6 100644 --- a/src/os/file_windows.go +++ b/src/os/file_windows.go @@ -103,20 +103,9 @@ func openFileNolog(name string, flag int, perm FileMode) (*File, error) { return nil, &PathError{Op: "open", Path: name, Err: syscall.ENOENT} } path := fixLongPath(name) - r, e := syscall.Open(path, flag|syscall.O_CLOEXEC, syscallMode(perm)) - if e != nil { - // We should return EISDIR when we are trying to open a directory with write access. - if e == syscall.ERROR_ACCESS_DENIED && (flag&O_WRONLY != 0 || flag&O_RDWR != 0) { - pathp, e1 := syscall.UTF16PtrFromString(path) - if e1 == nil { - var fa syscall.Win32FileAttributeData - e1 = syscall.GetFileAttributesEx(pathp, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fa))) - if e1 == nil && fa.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - e = syscall.EISDIR - } - } - } - return nil, &PathError{Op: "open", Path: name, Err: e} + r, err := syscall.Open(path, flag|syscall.O_CLOEXEC, syscallMode(perm)) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} } return newFile(r, name, "file"), nil } @@ -234,17 +223,13 @@ func Pipe() (r *File, w *File, err error) { return newFile(p[0], "|0", "pipe"), newFile(p[1], "|1", "pipe"), nil } -var ( - useGetTempPath2Once sync.Once - useGetTempPath2 bool -) +var useGetTempPath2 = sync.OnceValue(func() bool { + return windows.ErrorLoadingGetTempPath2() == nil +}) func tempDir() string { - useGetTempPath2Once.Do(func() { - useGetTempPath2 = (windows.ErrorLoadingGetTempPath2() == nil) - }) getTempPath := syscall.GetTempPath - if useGetTempPath2 { + if useGetTempPath2() { getTempPath = windows.GetTempPath2 } n := uint32(syscall.MAX_PATH) @@ -430,10 +415,13 @@ func readReparseLink(path string) (string, error) { return "", err } defer syscall.CloseHandle(h) + return readReparseLinkHandle(h) +} +func readReparseLinkHandle(h syscall.Handle) (string, error) { rdbbuf := make([]byte, syscall.MAXIMUM_REPARSE_DATA_BUFFER_SIZE) var bytesReturned uint32 - err = syscall.DeviceIoControl(h, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil) + err := syscall.DeviceIoControl(h, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil) if err != nil { return "", err } diff --git a/src/os/getwd.go b/src/os/getwd.go index 90604cf2..5ce948fa 100644 --- a/src/os/getwd.go +++ b/src/os/getwd.go @@ -15,45 +15,61 @@ var getwdCache struct { dir string } -// Getwd returns a rooted path name corresponding to the +// Getwd returns an absolute path name corresponding to the // current directory. If the current directory can be // reached via multiple paths (due to symbolic links), // Getwd may return any one of them. +// +// On Unix platforms, if the environment variable PWD +// provides an absolute name, and it is a name of the +// current directory, it is returned. func Getwd() (dir string, err error) { if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { - return syscall.Getwd() + // Use syscall.Getwd directly for + // - plan9: see reasons in CL 89575; + // - windows: syscall implementation is sufficient, + // and we should not rely on $PWD. + dir, err = syscall.Getwd() + return dir, NewSyscallError("getwd", err) } // Clumsy but widespread kludge: // if $PWD is set and matches ".", use it. - dot, err := statNolog(".") - if err != nil { - return "", err - } + var dot FileInfo dir = Getenv("PWD") if len(dir) > 0 && dir[0] == '/' { + dot, err = statNolog(".") + if err != nil { + return "", err + } d, err := statNolog(dir) if err == nil && SameFile(dot, d) { return dir, nil } + // If err is ENAMETOOLONG here, the syscall.Getwd below will + // fail with the same error, too, but let's give it a try + // anyway as the fallback code is much slower. } // If the operating system provides a Getwd call, use it. - // Otherwise, we're trying to find our way back to ".". if syscall.ImplementsGetwd { - var ( - s string - e error - ) - for { - s, e = syscall.Getwd() - if e != syscall.EINTR { - break - } + dir, err = ignoringEINTR2(syscall.Getwd) + // Linux returns ENAMETOOLONG if the result is too long. + // Some BSD systems appear to return EINVAL. + // FreeBSD systems appear to use ENOMEM + // Solaris appears to use ERANGE. + if err != syscall.ENAMETOOLONG && err != syscall.EINVAL && err != errERANGE && err != errENOMEM { + return dir, NewSyscallError("getwd", err) } - return s, NewSyscallError("getwd", e) } + // We're trying to find our way back to ".". + if dot == nil { + dot, err = statNolog(".") + if err != nil { + return "", err + } + } // Apply same kludge but to cached dir instead of $PWD. getwdCache.Lock() dir = getwdCache.dir @@ -82,9 +98,9 @@ func Getwd() (dir string, err error) { dir = "" for parent := ".."; ; parent = "../" + parent { if len(parent) >= 1024 { // Sanity check - return "", syscall.ENAMETOOLONG + return "", NewSyscallError("getwd", syscall.ENAMETOOLONG) } - fd, err := openFileNolog(parent, O_RDONLY, 0) + fd, err := openDirNolog(parent) if err != nil { return "", err } @@ -93,7 +109,14 @@ func Getwd() (dir string, err error) { names, err := fd.Readdirnames(100) if err != nil { fd.Close() - return "", err + // Readdirnames can return io.EOF or other error. + // In any case, we're here because syscall.Getwd + // is not implemented or failed with ENAMETOOLONG, + // so return the most sensible error. + if syscall.ImplementsGetwd { + return "", NewSyscallError("getwd", syscall.ENAMETOOLONG) + } + return "", NewSyscallError("getwd", errENOSYS) } for _, name := range names { d, _ := lstatNolog(parent + "/" + name) diff --git a/src/os/getwd_unix_test.go b/src/os/getwd_unix_test.go new file mode 100644 index 00000000..3f3067b5 --- /dev/null +++ b/src/os/getwd_unix_test.go @@ -0,0 +1,113 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package os_test + +import ( + "errors" + . "os" + "path/filepath" + "runtime" + "strings" + "syscall" + "testing" +) + +func TestGetwdDeep(t *testing.T) { + testGetwdDeep(t, false) +} + +func TestGetwdDeepWithPWDSet(t *testing.T) { + testGetwdDeep(t, true) +} + +// testGetwdDeep checks that os.Getwd is able to return paths +// longer than syscall.PathMax (with or without PWD set). +func testGetwdDeep(t *testing.T, setPWD bool) { + tempDir := t.TempDir() + + dir := tempDir + t.Chdir(dir) + + if setPWD { + t.Setenv("PWD", dir) + } else { + // When testing os.Getwd, setting PWD to empty string + // is the same as unsetting it, but the latter would + // be more complicated since we don't have t.Unsetenv. + t.Setenv("PWD", "") + } + + name := strings.Repeat("a", 200) + for { + if err := Mkdir(name, 0o700); err != nil { + t.Fatal(err) + } + if err := Chdir(name); err != nil { + t.Fatal(err) + } + if setPWD { + dir += "/" + name + if err := Setenv("PWD", dir); err != nil { + t.Fatal(err) + } + t.Logf(" $PWD len: %d", len(dir)) + } + + wd, err := Getwd() + t.Logf("Getwd len: %d", len(wd)) + if err != nil { + // We can get an EACCES error if we can't read up + // to root, which happens on the Android builders. + if errors.Is(err, syscall.EACCES) { + t.Logf("ignoring EACCES error: %v", err) + break + } + t.Fatal(err) + } + if setPWD && wd != dir { + // It's possible for the stat of PWD to fail + // with ENAMETOOLONG, and for getwd to fail for + // the same reason, and it's possible for $TMPDIR + // to contain a symlink. In that case the fallback + // code will not return the same directory. + if len(dir) > 1000 { + symDir, err := filepath.EvalSymlinks(tempDir) + if err == nil && symDir != tempDir { + t.Logf("EvalSymlinks(%q) = %q", tempDir, symDir) + if strings.Replace(dir, tempDir, symDir, 1) == wd { + // Symlink confusion is OK. + break + } + } + } + + t.Fatalf("Getwd: got %q, want same value as $PWD: %q", wd, dir) + } + // Ideally the success criterion should be len(wd) > syscall.PathMax, + // but the latter is not public for some platforms, so use Stat(wd). + // When it fails with ENAMETOOLONG, it means: + // - wd is longer than PathMax; + // - Getwd have used the slow fallback code. + // + // To avoid an endless loop here in case Stat keeps working, + // check if len(wd) is above the largest known PathMax among + // all Unix platforms (4096, on Linux). + if _, err := Stat(wd); err != nil || len(wd) > 4096 { + t.Logf("Done; len(wd)=%d", len(wd)) + // Most systems return ENAMETOOLONG. + // Dragonfly returns EFAULT. + switch { + case err == nil: + case errors.Is(err, syscall.ENAMETOOLONG): + case runtime.GOOS == "dragonfly" && errors.Is(err, syscall.EFAULT): + default: + t.Fatalf("unexpected Stat error: %v", err) + } + break + } + } +} diff --git a/src/os/os_test.go b/src/os/os_test.go index f1755dfa..1e2db94d 100644 --- a/src/os/os_test.go +++ b/src/os/os_test.go @@ -16,7 +16,6 @@ import ( . "os" "os/exec" "path/filepath" - "reflect" "runtime" "runtime/debug" "slices" @@ -196,7 +195,7 @@ func TestStat(t *testing.T) { } func TestStatError(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) path := "no-such-file" @@ -233,8 +232,7 @@ func TestStatError(t *testing.T) { func TestStatSymlinkLoop(t *testing.T) { testenv.MustHaveSymlink(t) - - defer chtmpdir(t)() + t.Chdir(t.TempDir()) err := Symlink("x", "y") if err != nil { @@ -805,13 +803,13 @@ func TestReaddirStatFailures(t *testing.T) { } if got, want := names(mustReadDir("initial readdir")), - []string{"good1", "good2", "x"}; !reflect.DeepEqual(got, want) { + []string{"good1", "good2", "x"}; !slices.Equal(got, want) { t.Errorf("initial readdir got %q; want %q", got, want) } xerr = ErrNotExist if got, want := names(mustReadDir("with x disappearing")), - []string{"good1", "good2"}; !reflect.DeepEqual(got, want) { + []string{"good1", "good2"}; !slices.Equal(got, want) { t.Errorf("with x disappearing, got %q; want %q", got, want) } @@ -852,8 +850,8 @@ func TestReaddirOfFile(t *testing.T) { func TestHardLink(t *testing.T) { testenv.MustHaveLink(t) + t.Chdir(t.TempDir()) - defer chtmpdir(t)() from, to := "hardlinktestfrom", "hardlinktestto" file, err := Create(to) if err != nil { @@ -908,32 +906,10 @@ func TestHardLink(t *testing.T) { } } -// chtmpdir changes the working directory to a new temporary directory and -// provides a cleanup function. -func chtmpdir(t *testing.T) func() { - oldwd, err := Getwd() - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - d, err := MkdirTemp("", "test") - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - if err := Chdir(d); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - return func() { - if err := Chdir(oldwd); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - RemoveAll(d) - } -} - func TestSymlink(t *testing.T) { testenv.MustHaveSymlink(t) + t.Chdir(t.TempDir()) - defer chtmpdir(t)() from, to := "symlinktestfrom", "symlinktestto" file, err := Create(to) if err != nil { @@ -993,8 +969,8 @@ func TestSymlink(t *testing.T) { func TestLongSymlink(t *testing.T) { testenv.MustHaveSymlink(t) + t.Chdir(t.TempDir()) - defer chtmpdir(t)() s := "0123456789abcdef" // Long, but not too long: a common limit is 255. s = s + s + s + s + s + s + s + s + s + s + s + s + s + s + s @@ -1013,7 +989,7 @@ func TestLongSymlink(t *testing.T) { } func TestRename(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" file, err := Create(from) @@ -1034,7 +1010,7 @@ func TestRename(t *testing.T) { } func TestRenameOverwriteDest(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" toData := []byte("to") @@ -1071,7 +1047,7 @@ func TestRenameOverwriteDest(t *testing.T) { } func TestRenameFailed(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" err := Rename(from, to) @@ -1094,7 +1070,7 @@ func TestRenameFailed(t *testing.T) { } func TestRenameNotExisting(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "doesnt-exist", "dest" Mkdir(to, 0777) @@ -1105,7 +1081,7 @@ func TestRenameNotExisting(t *testing.T) { } func TestRenameToDirFailed(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" Mkdir(from, 0777) @@ -1150,7 +1126,7 @@ func TestRenameCaseDifference(pt *testing.T) { for _, test := range tests { pt.Run(test.name, func(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) if err := test.create(); err != nil { t.Fatalf("failed to create test file: %s", err) @@ -1530,10 +1506,10 @@ func testChtimes(t *testing.T, name string) { t.Log(errormsg) t.Log("Known NetBSD issue (atime not changed on fs mounted with noatime); ignoring.") } else { - t.Errorf(errormsg) + t.Error(errormsg) } default: - t.Errorf(errormsg) + t.Error(errormsg) } } @@ -1572,7 +1548,7 @@ func TestFileChdir(t *testing.T) { if err != nil { t.Fatalf("Getwd: %s", err) } - defer Chdir(wd) + t.Chdir(".") // Ensure wd is restored after the test. fd, err := Open(".") if err != nil { @@ -1607,10 +1583,8 @@ func TestFileChdir(t *testing.T) { } func TestChdirAndGetwd(t *testing.T) { - fd, err := Open(".") - if err != nil { - t.Fatalf("Open .: %s", err) - } + t.Chdir(t.TempDir()) // Ensure wd is restored after the test. + // These are chosen carefully not to be symlinks on a Mac // (unlike, say, /var, /etc), except /tmp, which we handle below. dirs := []string{"/", "/usr/bin", "/tmp"} @@ -1624,16 +1598,16 @@ func TestChdirAndGetwd(t *testing.T) { dirs = nil for _, dir := range []string{t.TempDir(), t.TempDir()} { // Expand symlinks so path equality tests work. - dir, err = filepath.EvalSymlinks(dir) + dir, err := filepath.EvalSymlinks(dir) if err != nil { t.Fatalf("EvalSymlinks: %v", err) } dirs = append(dirs, dir) } } - oldwd := Getenv("PWD") for mode := 0; mode < 2; mode++ { for _, d := range dirs { + var err error if mode == 0 { err = Chdir(d) } else { @@ -1649,30 +1623,17 @@ func TestChdirAndGetwd(t *testing.T) { Setenv("PWD", "/tmp") } pwd, err1 := Getwd() - Setenv("PWD", oldwd) - err2 := fd.Chdir() - if err2 != nil { - // We changed the current directory and cannot go back. - // Don't let the tests continue; they'll scribble - // all over some other directory. - fmt.Fprintf(Stderr, "fchdir back to dot failed: %s\n", err2) - Exit(1) - } if err != nil { - fd.Close() t.Fatalf("Chdir %s: %s", d, err) } if err1 != nil { - fd.Close() t.Fatalf("Getwd in %s: %s", d, err1) } if !equal(pwd, d) { - fd.Close() t.Fatalf("Getwd returned %q want %q", pwd, d) } } } - fd.Close() } // Test that Chdir+Getwd is program-wide. @@ -1683,17 +1644,7 @@ func TestProgWideChdir(t *testing.T) { done := make(chan struct{}) d := t.TempDir() - oldwd, err := Getwd() - if err != nil { - t.Fatalf("Getwd: %v", err) - } - defer func() { - if err := Chdir(oldwd); err != nil { - // It's not safe to continue with tests if we can't get back to - // the original working directory. - panic(err) - } - }() + t.Chdir(d) // Note the deferred Wait must be called after the deferred close(done), // to ensure the N goroutines have been released even if the main goroutine @@ -1748,6 +1699,7 @@ func TestProgWideChdir(t *testing.T) { } }(i) } + var err error if err = Chdir(d); err != nil { t.Fatalf("Chdir: %v", err) } @@ -1793,13 +1745,6 @@ func TestSeek(t *testing.T) { for i, tt := range tests { off, err := f.Seek(tt.in, tt.whence) if off != tt.out || err != nil { - if e, ok := err.(*PathError); ok && e.Err == syscall.EINVAL && tt.out > 1<<32 && runtime.GOOS == "linux" { - mounts, _ := ReadFile("/proc/mounts") - if strings.Contains(string(mounts), "reiserfs") { - // Reiserfs rejects the big seeks. - t.Skipf("skipping test known to fail on reiserfs; https://golang.org/issue/91") - } - } t.Errorf("#%d: Seek(%v, %v) = %v, %v want %v, nil", i, tt.in, tt.whence, off, err, tt.out) } } @@ -1832,43 +1777,62 @@ func TestSeekError(t *testing.T) { } } -type openErrorTest struct { - path string - mode int - error error -} - -var openErrorTests = []openErrorTest{ - { - sfdir + "/no-such-file", - O_RDONLY, - syscall.ENOENT, - }, - { - sfdir, - O_WRONLY, - syscall.EISDIR, - }, - { - sfdir + "/" + sfname + "/no-such-file", - O_WRONLY, - syscall.ENOTDIR, - }, -} - func TestOpenError(t *testing.T) { t.Parallel() - - for _, tt := range openErrorTests { - f, err := OpenFile(tt.path, tt.mode, 0) + dir := makefs(t, []string{ + "is-a-file", + "is-a-dir/", + }) + t.Run("NoRoot", func(t *testing.T) { testOpenError(t, dir, false) }) + t.Run("InRoot", func(t *testing.T) { testOpenError(t, dir, true) }) +} +func testOpenError(t *testing.T, dir string, rooted bool) { + t.Parallel() + var r *Root + if rooted { + var err error + r, err = OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + } + for _, tt := range []struct { + path string + mode int + error error + }{{ + "no-such-file", + O_RDONLY, + syscall.ENOENT, + }, { + "is-a-dir", + O_WRONLY, + syscall.EISDIR, + }, { + "is-a-file/no-such-file", + O_WRONLY, + syscall.ENOTDIR, + }} { + var f *File + var err error + var name string + if rooted { + name = fmt.Sprintf("Root(%q).OpenFile(%q, %d)", dir, tt.path, tt.mode) + f, err = r.OpenFile(tt.path, tt.mode, 0) + } else { + path := filepath.Join(dir, tt.path) + name = fmt.Sprintf("OpenFile(%q, %d)", path, tt.mode) + f, err = OpenFile(path, tt.mode, 0) + } if err == nil { - t.Errorf("Open(%q, %d) succeeded", tt.path, tt.mode) + t.Errorf("%v succeeded", name) f.Close() continue } perr, ok := err.(*PathError) if !ok { - t.Errorf("Open(%q, %d) returns error of %T type; want *PathError", tt.path, tt.mode, err) + t.Errorf("%v returns error of %T type; want *PathError", name, err) } if perr.Err != tt.error { if runtime.GOOS == "plan9" { @@ -1876,12 +1840,14 @@ func TestOpenError(t *testing.T) { expectedErrStr := strings.Replace(tt.error.Error(), "file ", "", 1) if !strings.HasSuffix(syscallErrStr, expectedErrStr) { // Some Plan 9 file servers incorrectly return - // EACCES rather than EISDIR when a directory is + // EPERM or EACCES rather than EISDIR when a directory is // opened for write. - if tt.error == syscall.EISDIR && strings.HasSuffix(syscallErrStr, syscall.EACCES.Error()) { + if tt.error == syscall.EISDIR && + (strings.HasSuffix(syscallErrStr, syscall.EPERM.Error()) || + strings.HasSuffix(syscallErrStr, syscall.EACCES.Error())) { continue } - t.Errorf("Open(%q, %d) = _, %q; want suffix %q", tt.path, tt.mode, syscallErrStr, expectedErrStr) + t.Errorf("%v = _, %q; want suffix %q", name, syscallErrStr, expectedErrStr) } continue } @@ -1892,7 +1858,7 @@ func TestOpenError(t *testing.T) { continue } } - t.Errorf("Open(%q, %d) = _, %q; want %q", tt.path, tt.mode, perr.Err.Error(), tt.error.Error()) + t.Errorf("%v = _, %q; want %q", name, perr.Err.Error(), tt.error.Error()) } } } @@ -2111,7 +2077,7 @@ func TestWriteAtNegativeOffset(t *testing.T) { // Verify that WriteAt doesn't work in append mode. func TestWriteAtInAppendMode(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) f, err := OpenFile("write_at_in_append_mode.txt", O_APPEND|O_CREATE, 0666) if err != nil { t.Fatalf("OpenFile: %v", err) @@ -2124,8 +2090,15 @@ func TestWriteAtInAppendMode(t *testing.T) { } } -func writeFile(t *testing.T, fname string, flag int, text string) string { - f, err := OpenFile(fname, flag, 0666) +func writeFile(t *testing.T, r *Root, fname string, flag int, text string) string { + t.Helper() + var f *File + var err error + if r == nil { + f, err = OpenFile(fname, flag, 0666) + } else { + f, err = r.OpenFile(fname, flag, 0666) + } if err != nil { t.Fatalf("Open: %v", err) } @@ -2142,35 +2115,180 @@ func writeFile(t *testing.T, fname string, flag int, text string) string { } func TestAppend(t *testing.T) { - defer chtmpdir(t)() - const f = "append.txt" - s := writeFile(t, f, O_CREATE|O_TRUNC|O_RDWR, "new") - if s != "new" { - t.Fatalf("writeFile: have %q want %q", s, "new") + testMaybeRooted(t, func(t *testing.T, r *Root) { + const f = "append.txt" + s := writeFile(t, r, f, O_CREATE|O_TRUNC|O_RDWR, "new") + if s != "new" { + t.Fatalf("writeFile: have %q want %q", s, "new") + } + s = writeFile(t, r, f, O_APPEND|O_RDWR, "|append") + if s != "new|append" { + t.Fatalf("writeFile: have %q want %q", s, "new|append") + } + s = writeFile(t, r, f, O_CREATE|O_APPEND|O_RDWR, "|append") + if s != "new|append|append" { + t.Fatalf("writeFile: have %q want %q", s, "new|append|append") + } + err := Remove(f) + if err != nil { + t.Fatalf("Remove: %v", err) + } + s = writeFile(t, r, f, O_CREATE|O_APPEND|O_RDWR, "new&append") + if s != "new&append" { + t.Fatalf("writeFile: after append have %q want %q", s, "new&append") + } + s = writeFile(t, r, f, O_CREATE|O_RDWR, "old") + if s != "old&append" { + t.Fatalf("writeFile: after create have %q want %q", s, "old&append") + } + s = writeFile(t, r, f, O_CREATE|O_TRUNC|O_RDWR, "new") + if s != "new" { + t.Fatalf("writeFile: after truncate have %q want %q", s, "new") + } + }) +} + +// TestFilePermissions tests setting Unix permission bits on file creation. +func TestFilePermissions(t *testing.T) { + if Getuid() == 0 { + t.Skip("skipping test when running as root") } - s = writeFile(t, f, O_APPEND|O_RDWR, "|append") - if s != "new|append" { - t.Fatalf("writeFile: have %q want %q", s, "new|append") + for _, test := range []struct { + name string + mode FileMode + }{ + {"r", 0o444}, + {"w", 0o222}, + {"rw", 0o666}, + } { + t.Run(test.name, func(t *testing.T) { + switch runtime.GOOS { + case "windows": + if test.mode&0444 == 0 { + t.Skip("write-only files not supported on " + runtime.GOOS) + } + case "wasip1": + t.Skip("file permissions not supported on " + runtime.GOOS) + } + testMaybeRooted(t, func(t *testing.T, r *Root) { + const filename = "f" + var f *File + var err error + if r == nil { + f, err = OpenFile(filename, O_RDWR|O_CREATE|O_EXCL, test.mode) + } else { + f, err = r.OpenFile(filename, O_RDWR|O_CREATE|O_EXCL, test.mode) + } + if err != nil { + t.Fatal(err) + } + f.Close() + b, err := ReadFile(filename) + if test.mode&0o444 != 0 { + if err != nil { + t.Errorf("ReadFile = %v; want success", err) + } + } else { + if err == nil { + t.Errorf("ReadFile = %q, ; want failure", string(b)) + } + } + _, err = Stat(filename) + if err != nil { + t.Errorf("Stat = %v; want success", err) + } + err = WriteFile(filename, nil, 0666) + if test.mode&0o222 != 0 { + if err != nil { + t.Errorf("WriteFile = %v; want success", err) + b, err := ReadFile(filename) + t.Errorf("ReadFile: %v", err) + t.Errorf("file contents: %q", b) + } + } else { + if err == nil { + t.Errorf("WriteFile(%q) = ; want failure", filename) + st, err := Stat(filename) + if err == nil { + t.Errorf("mode: %s", st.Mode()) + } + b, err := ReadFile(filename) + t.Errorf("ReadFile: %v", err) + t.Errorf("file contents: %q", b) + } + } + }) + }) } - s = writeFile(t, f, O_CREATE|O_APPEND|O_RDWR, "|append") - if s != "new|append|append" { - t.Fatalf("writeFile: have %q want %q", s, "new|append|append") - } - err := Remove(f) - if err != nil { - t.Fatalf("Remove: %v", err) - } - s = writeFile(t, f, O_CREATE|O_APPEND|O_RDWR, "new&append") - if s != "new&append" { - t.Fatalf("writeFile: after append have %q want %q", s, "new&append") - } - s = writeFile(t, f, O_CREATE|O_RDWR, "old") - if s != "old&append" { - t.Fatalf("writeFile: after create have %q want %q", s, "old&append") - } - s = writeFile(t, f, O_CREATE|O_TRUNC|O_RDWR, "new") - if s != "new" { - t.Fatalf("writeFile: after truncate have %q want %q", s, "new") + +} + +// TestFileRDWRFlags tests the O_RDONLY, O_WRONLY, and O_RDWR flags. +func TestFileRDWRFlags(t *testing.T) { + for _, test := range []struct { + name string + flag int + }{ + {"O_RDONLY", O_RDONLY}, + {"O_WRONLY", O_WRONLY}, + {"O_RDWR", O_RDWR}, + } { + t.Run(test.name, func(t *testing.T) { + testMaybeRooted(t, func(t *testing.T, r *Root) { + const filename = "f" + content := []byte("content") + if err := WriteFile(filename, content, 0666); err != nil { + t.Fatal(err) + } + var f *File + var err error + if r == nil { + f, err = OpenFile(filename, test.flag, 0) + } else { + f, err = r.OpenFile(filename, test.flag, 0) + } + if err != nil { + t.Fatal(err) + } + defer f.Close() + got, err := io.ReadAll(f) + if test.flag == O_WRONLY { + if err == nil { + t.Errorf("read file: %q, %v; want error", got, err) + } + } else { + if err != nil || !bytes.Equal(got, content) { + t.Errorf("read file: %q, %v; want %q, ", got, err, content) + } + } + if _, err := f.Seek(0, 0); err != nil { + t.Fatalf("f.Seek: %v", err) + } + newcontent := []byte("CONTENT") + _, err = f.Write(newcontent) + if test.flag == O_RDONLY { + if err == nil { + t.Errorf("write file: succeeded, want error") + } + } else { + if err != nil { + t.Errorf("write file: %v, want success", err) + } + } + f.Close() + got, err = ReadFile(filename) + if err != nil { + t.Fatal(err) + } + want := content + if test.flag != O_RDONLY { + want = newcontent + } + if !bytes.Equal(got, want) { + t.Fatalf("after write, file contains %q, want %q", got, want) + } + }) + }) } } @@ -2201,7 +2319,7 @@ func TestNilProcessStateString(t *testing.T) { } func TestSameFile(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) fa, err := Create("a") if err != nil { t.Fatalf("Create(a): %v", err) @@ -2344,13 +2462,8 @@ func TestStatStdin(t *testing.T) { Exit(0) } - exe, err := Executable() - if err != nil { - t.Skipf("can't find executable: %v", err) - } - - testenv.MustHaveExec(t) t.Parallel() + exe := testenv.Executable(t) fi, err := Stdin.Stat() if err != nil { @@ -2516,11 +2629,10 @@ func TestLongPath(t *testing.T) { } func testKillProcess(t *testing.T, processKiller func(p *Process)) { - testenv.MustHaveExec(t) t.Parallel() // Re-exec the test binary to start a process that hangs until stdin is closed. - cmd := testenv.Command(t, Args[0]) + cmd := testenv.Command(t, testenv.Executable(t)) cmd.Env = append(cmd.Environ(), "GO_OS_TEST_DRAIN_STDIN=1") stdout, err := cmd.StdoutPipe() if err != nil { @@ -2569,10 +2681,9 @@ func TestGetppid(t *testing.T) { Exit(0) } - testenv.MustHaveExec(t) t.Parallel() - cmd := testenv.Command(t, Args[0], "-test.run=^TestGetppid$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestGetppid$") cmd.Env = append(Environ(), "GO_WANT_HELPER_PROCESS=1") // verify that Getppid() from the forked process reports our process id @@ -2665,10 +2776,7 @@ func TestRemoveAllRace(t *testing.T) { n := runtime.GOMAXPROCS(16) defer runtime.GOMAXPROCS(n) - root, err := MkdirTemp("", "issue") - if err != nil { - t.Fatal(err) - } + root := t.TempDir() mkdirTree(t, root, 1, 6) hold := make(chan struct{}) var wg sync.WaitGroup @@ -2690,6 +2798,8 @@ func TestRemoveAllRace(t *testing.T) { // Test that reading from a pipe doesn't use up a thread. func TestPipeThreads(t *testing.T) { switch runtime.GOOS { + case "aix": + t.Skip("skipping on aix; issue 70131") case "illumos", "solaris": t.Skip("skipping on Solaris and illumos; issue 19111") case "windows": @@ -2704,11 +2814,6 @@ func TestPipeThreads(t *testing.T) { threads := 100 - // OpenBSD has a low default for max number of files. - if runtime.GOOS == "openbsd" { - threads = 50 - } - r := make([]*File, threads) w := make([]*File, threads) for i := 0; i < threads; i++ { @@ -2813,6 +2918,33 @@ func TestUserCacheDir(t *testing.T) { } } +func TestUserCacheDirXDGConfigDirEnvVar(t *testing.T) { + switch runtime.GOOS { + case "windows", "darwin", "plan9": + t.Skip("$XDG_CACHE_HOME is effective only on Unix systems") + } + + wd, err := Getwd() + if err != nil { + t.Fatal(err) + } + t.Setenv("XDG_CACHE_HOME", wd) + + dir, err := UserCacheDir() + if err != nil { + t.Fatal(err) + } + if dir != wd { + t.Fatalf("UserCacheDir returned %q; want the value of $XDG_CACHE_HOME %q", dir, wd) + } + + t.Setenv("XDG_CACHE_HOME", "some-dir") + _, err = UserCacheDir() + if err == nil { + t.Fatal("UserCacheDir succeeded though $XDG_CACHE_HOME contains a relative path") + } +} + func TestUserConfigDir(t *testing.T) { t.Parallel() @@ -2837,6 +2969,33 @@ func TestUserConfigDir(t *testing.T) { } } +func TestUserConfigDirXDGConfigDirEnvVar(t *testing.T) { + switch runtime.GOOS { + case "windows", "darwin", "plan9": + t.Skip("$XDG_CONFIG_HOME is effective only on Unix systems") + } + + wd, err := Getwd() + if err != nil { + t.Fatal(err) + } + t.Setenv("XDG_CONFIG_HOME", wd) + + dir, err := UserConfigDir() + if err != nil { + t.Fatal(err) + } + if dir != wd { + t.Fatalf("UserConfigDir returned %q; want the value of $XDG_CONFIG_HOME %q", dir, wd) + } + + t.Setenv("XDG_CONFIG_HOME", "some-dir") + _, err = UserConfigDir() + if err == nil { + t.Fatal("UserConfigDir succeeded though $XDG_CONFIG_HOME contains a relative path") + } +} + func TestUserHomeDir(t *testing.T) { t.Parallel() @@ -2950,6 +3109,22 @@ func isDeadlineExceeded(err error) bool { // Test that opening a file does not change its permissions. Issue 38225. func TestOpenFileKeepsPermissions(t *testing.T) { + t.Run("OpenFile", func(t *testing.T) { + testOpenFileKeepsPermissions(t, OpenFile) + }) + t.Run("RootOpenFile", func(t *testing.T) { + testOpenFileKeepsPermissions(t, func(name string, flag int, perm FileMode) (*File, error) { + dir, file := filepath.Split(name) + r, err := OpenRoot(dir) + if err != nil { + return nil, err + } + defer r.Close() + return r.OpenFile(file, flag, perm) + }) + }) +} +func testOpenFileKeepsPermissions(t *testing.T, openf func(name string, flag int, perm FileMode) (*File, error)) { t.Parallel() dir := t.TempDir() @@ -2961,7 +3136,7 @@ func TestOpenFileKeepsPermissions(t *testing.T) { if err := f.Close(); err != nil { t.Error(err) } - f, err = OpenFile(name, O_WRONLY|O_CREATE|O_TRUNC, 0) + f, err = openf(name, O_WRONLY|O_CREATE|O_TRUNC, 0) if err != nil { t.Fatal(err) } @@ -3015,10 +3190,21 @@ func forceMFTUpdateOnWindows(t *testing.T, path string) { func TestDirFS(t *testing.T) { t.Parallel() + testDirFS(t, DirFS("./testdata/dirfs")) +} +func TestRootDirFS(t *testing.T) { + t.Parallel() + r, err := OpenRoot("./testdata/dirfs") + if err != nil { + t.Fatal(err) + } + testDirFS(t, r.FS()) +} + +func testDirFS(t *testing.T, fsys fs.FS) { forceMFTUpdateOnWindows(t, "./testdata/dirfs") - fsys := DirFS("./testdata/dirfs") if err := fstest.TestFS(fsys, "a", "b", "dir/x"); err != nil { t.Fatal(err) } @@ -3333,24 +3519,7 @@ func TestCopyFS(t *testing.T) { if err := fstest.TestFS(tmpFsys, "a", "b", "dir/x"); err != nil { t.Fatal("TestFS:", err) } - if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { - if d.IsDir() { - return nil - } - - data, err := fs.ReadFile(fsys, path) - if err != nil { - return err - } - newData, err := fs.ReadFile(tmpFsys, path) - if err != nil { - return err - } - if !bytes.Equal(data, newData) { - return errors.New("file " + path + " contents differ") - } - return nil - }); err != nil { + if err := verifyCopyFS(t, fsys, tmpFsys); err != nil { t.Fatal("comparing two directories:", err) } @@ -3379,24 +3548,7 @@ func TestCopyFS(t *testing.T) { if err := fstest.TestFS(tmpFsys, "william", "carl", "daVinci", "einstein", "dir/newton"); err != nil { t.Fatal("TestFS:", err) } - if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { - if d.IsDir() { - return nil - } - - data, err := fs.ReadFile(fsys, path) - if err != nil { - return err - } - newData, err := fs.ReadFile(tmpFsys, path) - if err != nil { - return err - } - if !bytes.Equal(data, newData) { - return errors.New("file " + path + " contents differ") - } - return nil - }); err != nil { + if err := verifyCopyFS(t, fsys, tmpFsys); err != nil { t.Fatal("comparing two directories:", err) } @@ -3409,15 +3561,109 @@ func TestCopyFS(t *testing.T) { } } +// verifyCopyFS checks the content and permission of each file inside copied FS to ensure +// the copied files satisfy the convention stipulated in CopyFS. +func verifyCopyFS(t *testing.T, originFS, copiedFS fs.FS) error { + testDir := filepath.Join(t.TempDir(), "test") + // umask doesn't apply to the wasip and windows and there is no general way to get masked perm, + // so create a dir and a file to compare the permission after umask if any + if err := Mkdir(testDir, ModePerm); err != nil { + return fmt.Errorf("mkdir %q failed: %v", testDir, err) + } + dirStat, err := Stat(testDir) + if err != nil { + return fmt.Errorf("stat dir %q failed: %v", testDir, err) + } + wantDirMode := dirStat.Mode() + + f, err := Create(filepath.Join(testDir, "tmp")) + if err != nil { + return fmt.Errorf("open %q failed: %v", filepath.Join(testDir, "tmp"), err) + } + defer f.Close() + wantFileRWStat, err := f.Stat() + if err != nil { + return fmt.Errorf("stat file %q failed: %v", f.Name(), err) + } + wantFileRWMode := wantFileRWStat.Mode() + + return fs.WalkDir(originFS, ".", func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + // the dir . is not the dir created by CopyFS so skip checking its permission + if d.Name() == "." { + return nil + } + + dinfo, err := fs.Stat(copiedFS, path) + if err != nil { + return err + } + + if dinfo.Mode() != wantDirMode { + return fmt.Errorf("dir %q mode is %v, want %v", + d.Name(), dinfo.Mode(), wantDirMode) + } + return nil + } + + fInfo, err := originFS.Open(path) + if err != nil { + return err + } + defer fInfo.Close() + copiedInfo, err := copiedFS.Open(path) + if err != nil { + return err + } + defer copiedInfo.Close() + + // verify the file contents are the same + data, err := io.ReadAll(fInfo) + if err != nil { + return err + } + newData, err := io.ReadAll(copiedInfo) + if err != nil { + return err + } + if !bytes.Equal(data, newData) { + return fmt.Errorf("file %q content is %s, want %s", path, newData, data) + } + + fStat, err := fInfo.Stat() + if err != nil { + return err + } + copiedStat, err := copiedInfo.Stat() + if err != nil { + return err + } + + // check whether the execute permission is inherited from original FS + + if copiedStat.Mode()&0111&wantFileRWMode != fStat.Mode()&0111&wantFileRWMode { + return fmt.Errorf("file %q execute mode is %v, want %v", + path, copiedStat.Mode()&0111, fStat.Mode()&0111) + } + + rwMode := copiedStat.Mode() &^ 0111 // unset the executable permission from file mode + if rwMode != wantFileRWMode { + return fmt.Errorf("file %q rw mode is %v, want %v", + path, rwMode, wantFileRWStat.Mode()) + } + return nil + }) +} + func TestCopyFSWithSymlinks(t *testing.T) { // Test it with absolute and relative symlinks that point inside and outside the tree. testenv.MustHaveSymlink(t) // Create a directory and file outside. tmpDir := t.TempDir() - outsideDir, err := MkdirTemp(tmpDir, "copyfs_out_") - if err != nil { - t.Fatalf("MkdirTemp: %v", err) + outsideDir := filepath.Join(tmpDir, "copyfs_out") + if err := Mkdir(outsideDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) } outsideFile := filepath.Join(outsideDir, "file.out.txt") @@ -3426,9 +3672,9 @@ func TestCopyFSWithSymlinks(t *testing.T) { } // Create a directory and file inside. - insideDir, err := MkdirTemp(tmpDir, "copyfs_in_") - if err != nil { - t.Fatalf("MkdirTemp: %v", err) + insideDir := filepath.Join(tmpDir, "copyfs_in") + if err := Mkdir(insideDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) } insideFile := filepath.Join(insideDir, "file.in.txt") if err := WriteFile(insideFile, []byte("Testing CopyFS inside"), 0644); err != nil { @@ -3474,9 +3720,9 @@ func TestCopyFSWithSymlinks(t *testing.T) { // Copy the directory tree and verify. forceMFTUpdateOnWindows(t, insideDir) fsys := DirFS(insideDir) - tmpDupDir, err := MkdirTemp(tmpDir, "copyfs_dup_") - if err != nil { - t.Fatalf("MkdirTemp: %v", err) + tmpDupDir := filepath.Join(tmpDir, "copyfs_dup") + if err := Mkdir(tmpDupDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) } // TODO(panjf2000): symlinks are currently not supported, and a specific error @@ -3548,3 +3794,57 @@ func TestCopyFSWithSymlinks(t *testing.T) { t.Fatal("comparing two directories:", err) } } + +func TestAppendDoesntOverwrite(t *testing.T) { + testMaybeRooted(t, func(t *testing.T, r *Root) { + name := "file" + if err := WriteFile(name, []byte("hello"), 0666); err != nil { + t.Fatal(err) + } + var f *File + var err error + if r == nil { + f, err = OpenFile(name, O_APPEND|O_WRONLY, 0) + } else { + f, err = r.OpenFile(name, O_APPEND|O_WRONLY, 0) + } + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte(" world")); err != nil { + f.Close() + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + got, err := ReadFile(name) + if err != nil { + t.Fatal(err) + } + want := "hello world" + if string(got) != want { + t.Fatalf("got %q, want %q", got, want) + } + }) +} + +func TestRemoveReadOnlyFile(t *testing.T) { + testMaybeRooted(t, func(t *testing.T, r *Root) { + if err := WriteFile("file", []byte("1"), 0); err != nil { + t.Fatal(err) + } + var err error + if r == nil { + err = Remove("file") + } else { + err = r.Remove("file") + } + if err != nil { + t.Fatalf("Remove read-only file: %v", err) + } + if _, err := Stat("file"); !IsNotExist(err) { + t.Fatalf("Stat read-only file after removal: %v (want IsNotExist)", err) + } + }) +} diff --git a/src/os/os_unix_test.go b/src/os/os_unix_test.go index fcc75e5e..7e728195 100644 --- a/src/os/os_unix_test.go +++ b/src/os/os_unix_test.go @@ -233,7 +233,9 @@ func TestMkdirStickyUmask(t *testing.T) { if runtime.GOOS == "wasip1" { t.Skip("file permissions not supported on " + runtime.GOOS) } - t.Parallel() + // Issue #69788: This test temporarily changes the umask for testing purposes, + // so it shouldn't be run in parallel with other test cases + // to avoid other tests (e.g., TestCopyFS) creating files with an unintended umask. const umask = 0077 dir := t.TempDir() @@ -372,7 +374,7 @@ func TestSplitPath(t *testing.T) { // // Regression test for go.dev/issue/60181 func TestIssue60181(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) want := "hello gopher" diff --git a/src/os/os_windows_test.go b/src/os/os_windows_test.go index caff0111..31c37901 100644 --- a/src/os/os_windows_test.go +++ b/src/os/os_windows_test.go @@ -17,7 +17,6 @@ import ( "os" "os/exec" "path/filepath" - "reflect" "runtime" "slices" "strings" @@ -33,28 +32,8 @@ var winreadlinkvolume = godebug.New("winreadlinkvolume") // For TestRawConnReadWrite. type syscallDescriptor = syscall.Handle -// chdir changes the current working directory to the named directory, -// and then restore the original working directory at the end of the test. -func chdir(t *testing.T, dir string) { - olddir, err := os.Getwd() - if err != nil { - t.Fatalf("chdir: %v", err) - } - if err := os.Chdir(dir); err != nil { - t.Fatalf("chdir %s: %v", dir, err) - } - - t.Cleanup(func() { - if err := os.Chdir(olddir); err != nil { - t.Errorf("chdir to original working directory %s: %v", olddir, err) - os.Exit(1) - } - }) -} - func TestSameWindowsFile(t *testing.T) { - temp := t.TempDir() - chdir(t, temp) + t.Chdir(t.TempDir()) f, err := os.Create("a") if err != nil { @@ -100,7 +79,7 @@ type dirLinkTest struct { func testDirLinks(t *testing.T, tests []dirLinkTest) { tmpdir := t.TempDir() - chdir(t, tmpdir) + t.Chdir(tmpdir) dir := filepath.Join(tmpdir, "dir") err := os.Mkdir(dir, 0777) @@ -459,7 +438,7 @@ func TestNetworkSymbolicLink(t *testing.T) { const _NERR_ServerNotStarted = syscall.Errno(2114) dir := t.TempDir() - chdir(t, dir) + t.Chdir(dir) pid := os.Getpid() shareName := fmt.Sprintf("GoSymbolicLinkTestShare%d", pid) @@ -562,8 +541,7 @@ func TestStatLxSymLink(t *testing.T) { t.Skip("skipping: WSL not detected") } - temp := t.TempDir() - chdir(t, temp) + t.Chdir(t.TempDir()) const target = "target" const link = "link" @@ -630,7 +608,7 @@ func TestBadNetPathError(t *testing.T) { } func TestStatDir(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) f, err := os.Open(".") if err != nil { @@ -660,7 +638,7 @@ func TestStatDir(t *testing.T) { func TestOpenVolumeName(t *testing.T) { tmpdir := t.TempDir() - chdir(t, tmpdir) + t.Chdir(tmpdir) want := []string{"file1", "file2", "file3", "gopher.txt"} slices.Sort(want) @@ -778,7 +756,7 @@ func TestReadStdin(t *testing.T) { for len(want) < 5 { want = append(want, "") } - if !reflect.DeepEqual(all, want) { + if !slices.Equal(all, want) { t.Errorf("reading %q:\nhave %x\nwant %x", s, all, want) } }) @@ -953,7 +931,9 @@ func findOneDriveDir() (string, error) { return "", fmt.Errorf("reading UserFolder failed: %v", err) } - if valtype == registry.EXPAND_SZ { + // REG_SZ values may also contain environment variables that need to be expanded. + // It's recommended but not required to use REG_EXPAND_SZ for paths that contain environment variables. + if valtype == registry.EXPAND_SZ || valtype == registry.SZ { expanded, err := registry.ExpandString(path) if err != nil { return "", fmt.Errorf("expanding UserFolder failed: %v", err) @@ -1012,6 +992,8 @@ func TestFileStatNUL(t *testing.T) { if err != nil { t.Fatal(err) } + defer f.Close() + fi, err := f.Stat() if err != nil { t.Fatal(err) @@ -1037,8 +1019,8 @@ func TestStatNUL(t *testing.T) { // works on Windows when developer mode is active. // This is supported starting Windows 10 (1703, v10.0.14972). func TestSymlinkCreation(t *testing.T) { - if !testenv.HasSymlink() && !isWindowsDeveloperModeActive() { - t.Skip("Windows developer mode is not active") + if !testenv.HasSymlink() { + t.Skip("skipping test; no symlink support") } t.Parallel() @@ -1054,23 +1036,6 @@ func TestSymlinkCreation(t *testing.T) { } } -// isWindowsDeveloperModeActive checks whether or not the developer mode is active on Windows 10. -// Returns false for prior Windows versions. -// see https://docs.microsoft.com/en-us/windows/uwp/get-started/enable-your-device-for-development -func isWindowsDeveloperModeActive() bool { - key, err := registry.OpenKey(registry.LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\AppModelUnlock", registry.READ) - if err != nil { - return false - } - - val, _, err := key.GetIntegerValue("AllowDevelopmentWithoutDevLicense") - if err != nil { - return false - } - - return val != 0 -} - // TestRootRelativeDirSymlink verifies that symlinks to paths relative to the // drive root (beginning with "\" but no volume name) are created with the // correct symlink type. @@ -1130,14 +1095,7 @@ func TestWorkingDirectoryRelativeSymlink(t *testing.T) { if err != nil { t.Fatal(err) } - defer func() { - if err := os.Chdir(oldwd); err != nil { - t.Fatal(err) - } - }() - if err := os.Chdir(temp); err != nil { - t.Fatal(err) - } + t.Chdir(temp) t.Logf("Chdir(%#q)", temp) wdRelDir := filepath.VolumeName(temp) + `dir\sub` // no backslash after volume. @@ -1222,10 +1180,7 @@ func TestRootDirAsTemp(t *testing.T) { testenv.MustHaveExec(t) t.Parallel() - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } + exe := testenv.Executable(t) newtmp, err := findUnusedDriveLetter() if err != nil { @@ -1301,6 +1256,9 @@ func TestReadlink(t *testing.T) { } t.Run(name, func(t *testing.T) { + if !tt.junction { + testenv.MustHaveSymlink(t) + } if !tt.relative { t.Parallel() } @@ -1328,7 +1286,7 @@ func TestReadlink(t *testing.T) { } else { want = relTarget } - chdir(t, tmpdir) + t.Chdir(tmpdir) link = filepath.Base(link) target = relTarget } else { diff --git a/src/os/path_windows_test.go b/src/os/path_windows_test.go index 0b5d7099..3fa02e2a 100644 --- a/src/os/path_windows_test.go +++ b/src/os/path_windows_test.go @@ -224,7 +224,7 @@ func TestRemoveAllLongPathRelative(t *testing.T) { // Test that RemoveAll doesn't hang with long relative paths. // See go.dev/issue/36375. tmp := t.TempDir() - chdir(t, tmp) + t.Chdir(tmp) dir := filepath.Join(tmp, "foo", "bar", strings.Repeat("a", 150), strings.Repeat("b", 150)) err := os.MkdirAll(dir, 0755) if err != nil { @@ -265,7 +265,7 @@ func TestLongPathAbs(t *testing.T) { } func TestLongPathRel(t *testing.T) { - chdir(t, t.TempDir()) + t.Chdir(t.TempDir()) target := strings.Repeat("b\\", 300) testLongPathAbs(t, target) diff --git a/src/os/pidfd_linux.go b/src/os/pidfd_linux.go index 0bfef775..fe4a743c 100644 --- a/src/os/pidfd_linux.go +++ b/src/os/pidfd_linux.go @@ -21,7 +21,7 @@ import ( "runtime" "sync" "syscall" - "unsafe" + _ "unsafe" // for linkname ) // ensurePidfd initializes the PidFD field in sysAttr if it is not already set. @@ -78,9 +78,6 @@ func pidfdFind(pid int) (uintptr, error) { return h, nil } -// _P_PIDFD is used as idtype argument to waitid syscall. -const _P_PIDFD = 3 - func (p *Process) pidfdWait() (*ProcessState, error) { // When pidfd is used, there is no wait/kill race (described in CL 23967) // because the PID recycle issue doesn't exist (IOW, pidfd, unlike PID, @@ -104,16 +101,12 @@ func (p *Process) pidfdWait() (*ProcessState, error) { var ( info unix.SiginfoChild rusage syscall.Rusage - e syscall.Errno ) - for { - _, _, e = syscall.Syscall6(syscall.SYS_WAITID, _P_PIDFD, handle, uintptr(unsafe.Pointer(&info)), syscall.WEXITED, uintptr(unsafe.Pointer(&rusage)), 0) - if e != syscall.EINTR { - break - } - } - if e != 0 { - return nil, NewSyscallError("waitid", e) + err := ignoringEINTR(func() error { + return unix.Waitid(unix.P_PIDFD, int(handle), &info, syscall.WEXITED, &rusage) + }) + if err != nil { + return nil, NewSyscallError("waitid", err) } // Release the Process' handle reference, in addition to the reference // we took above. @@ -168,12 +161,9 @@ func checkPidfd() error { defer syscall.Close(int(fd)) // Check waitid(P_PIDFD) works. - for { - _, _, err = syscall.Syscall6(syscall.SYS_WAITID, _P_PIDFD, fd, 0, syscall.WEXITED, 0, 0) - if err != syscall.EINTR { - break - } - } + err = ignoringEINTR(func() error { + return unix.Waitid(unix.P_PIDFD, int(fd), nil, syscall.WEXITED, nil) + }) // Expect ECHILD from waitid since we're not our own parent. if err != syscall.ECHILD { return NewSyscallError("pidfd_wait", err) diff --git a/src/os/pidfd_linux_test.go b/src/os/pidfd_linux_test.go index c1f41d02..6b10798d 100644 --- a/src/os/pidfd_linux_test.go +++ b/src/os/pidfd_linux_test.go @@ -93,11 +93,7 @@ func TestStartProcessWithPidfd(t *testing.T) { // Issue #69284 func TestPidfdLeak(t *testing.T) { - testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } + exe := testenv.Executable(t) // Find the next 10 descriptors. // We need to get more than one descriptor in practice; diff --git a/src/os/pidfd_other.go b/src/os/pidfd_other.go index ba9cbcb9..7a282307 100644 --- a/src/os/pidfd_other.go +++ b/src/os/pidfd_other.go @@ -20,8 +20,6 @@ func pidfdFind(_ int) (uintptr, error) { return 0, syscall.ENOSYS } -func (p *Process) pidfdRelease() {} - func (_ *Process) pidfdWait() (*ProcessState, error) { panic("unreachable") } diff --git a/src/os/read_test.go b/src/os/read_test.go index 1f79e89b..4659191d 100644 --- a/src/os/read_test.go +++ b/src/os/read_test.go @@ -78,16 +78,11 @@ func TestReadOnlyWriteFile(t *testing.T) { t.Parallel() // We don't want to use CreateTemp directly, since that opens a file for us as 0600. - tempDir, err := MkdirTemp("", t.Name()) - if err != nil { - t.Fatal(err) - } - defer RemoveAll(tempDir) - filename := filepath.Join(tempDir, "blurp.txt") + filename := filepath.Join(t.TempDir(), "blurp.txt") shmorp := []byte("shmorp") florp := []byte("florp") - err = WriteFile(filename, shmorp, 0444) + err := WriteFile(filename, shmorp, 0444) if err != nil { t.Fatalf("WriteFile %s: %v", filename, err) } diff --git a/src/os/readfrom_freebsd_test.go b/src/os/readfrom_freebsd_test.go new file mode 100644 index 00000000..18604995 --- /dev/null +++ b/src/os/readfrom_freebsd_test.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "internal/poll" + . "os" + "testing" +) + +var ( + copyFileTests = []copyFileTestFunc{newCopyFileRangeTest} + copyFileHooks = []copyFileTestHook{hookCopyFileRange} +) + +func testCopyFiles(t *testing.T, size, limit int64) { + testCopyFileRange(t, size, limit) +} + +func testCopyFileRange(t *testing.T, size int64, limit int64) { + dst, src, data, hook, name := newCopyFileRangeTest(t, size) + testCopyFile(t, dst, src, data, hook, limit, name) +} + +// newCopyFileRangeTest initializes a new test for copy_file_range. +// It hooks package os' call to poll.CopyFileRange and returns the hook, +// so it can be inspected. +func newCopyFileRangeTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileHook, name string) { + t.Helper() + + name = "newCopyFileRangeTest" + + dst, src, data = newCopyFileTest(t, size) + hook, _ = hookCopyFileRange(t) + + return +} + +func hookCopyFileRange(t *testing.T) (hook *copyFileHook, name string) { + name = "hookCopyFileRange" + + hook = new(copyFileHook) + orig := *PollCopyFileRangeP + t.Cleanup(func() { + *PollCopyFileRangeP = orig + }) + *PollCopyFileRangeP = func(dst, src *poll.FD, remain int64) (int64, bool, error) { + hook.called = true + hook.dstfd = dst.Sysfd + hook.srcfd = src.Sysfd + hook.written, hook.handled, hook.err = orig(dst, src, remain) + return hook.written, hook.handled, hook.err + } + return +} diff --git a/src/os/readfrom_linux_test.go b/src/os/readfrom_linux_test.go index 45867477..d33f9cf9 100644 --- a/src/os/readfrom_linux_test.go +++ b/src/os/readfrom_linux_test.go @@ -15,247 +15,12 @@ import ( . "os" "path/filepath" "strconv" - "strings" "sync" "syscall" "testing" "time" ) -func TestCopyFileRange(t *testing.T) { - sizes := []int{ - 1, - 42, - 1025, - syscall.Getpagesize() + 1, - 32769, - } - t.Run("Basic", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), -1) - }) - } - }) - t.Run("Limited", func(t *testing.T) { - t.Run("OneLess", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), int64(size)-1) - }) - } - }) - t.Run("Half", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), int64(size)/2) - }) - } - }) - t.Run("More", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), int64(size)+7) - }) - } - }) - }) - t.Run("DoesntTryInAppendMode", func(t *testing.T) { - dst, src, data, hook := newCopyFileRangeTest(t, 42) - - dst2, err := OpenFile(dst.Name(), O_RDWR|O_APPEND, 0755) - if err != nil { - t.Fatal(err) - } - defer dst2.Close() - - if _, err := io.Copy(dst2, src); err != nil { - t.Fatal(err) - } - if hook.called { - t.Fatal("called poll.CopyFileRange for destination in O_APPEND mode") - } - mustSeekStart(t, dst2) - mustContainData(t, dst2, data) // through traditional means - }) - t.Run("CopyFileItself", func(t *testing.T) { - hook := hookCopyFileRange(t) - - f, err := CreateTemp("", "file-readfrom-itself-test") - if err != nil { - t.Fatalf("failed to create tmp file: %v", err) - } - t.Cleanup(func() { - f.Close() - Remove(f.Name()) - }) - - data := []byte("hello world!") - if _, err := f.Write(data); err != nil { - t.Fatalf("failed to create and feed the file: %v", err) - } - - if err := f.Sync(); err != nil { - t.Fatalf("failed to save the file: %v", err) - } - - // Rewind it. - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to rewind the file: %v", err) - } - - // Read data from the file itself. - if _, err := io.Copy(f, f); err != nil { - t.Fatalf("failed to read from the file: %v", err) - } - - if !hook.called || hook.written != 0 || hook.handled || hook.err != nil { - t.Fatalf("poll.CopyFileRange should be called and return the EINVAL error, but got hook.called=%t, hook.err=%v", hook.called, hook.err) - } - - // Rewind it. - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to rewind the file: %v", err) - } - - data2, err := io.ReadAll(f) - if err != nil { - t.Fatalf("failed to read from the file: %v", err) - } - - // It should wind up a double of the original data. - if strings.Repeat(string(data), 2) != string(data2) { - t.Fatalf("data mismatch: %s != %s", string(data), string(data2)) - } - }) - t.Run("NotRegular", func(t *testing.T) { - t.Run("BothPipes", func(t *testing.T) { - hook := hookCopyFileRange(t) - - pr1, pw1, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr1.Close() - defer pw1.Close() - - pr2, pw2, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr2.Close() - defer pw2.Close() - - // The pipe is empty, and PIPE_BUF is large enough - // for this, by (POSIX) definition, so there is no - // need for an additional goroutine. - data := []byte("hello") - if _, err := pw1.Write(data); err != nil { - t.Fatal(err) - } - pw1.Close() - - n, err := io.Copy(pw2, pr1) - if err != nil { - t.Fatal(err) - } - if n != int64(len(data)) { - t.Fatalf("transferred %d, want %d", n, len(data)) - } - if !hook.called { - t.Fatalf("should have called poll.CopyFileRange") - } - pw2.Close() - mustContainData(t, pr2, data) - }) - t.Run("DstPipe", func(t *testing.T) { - dst, src, data, hook := newCopyFileRangeTest(t, 255) - dst.Close() - - pr, pw, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr.Close() - defer pw.Close() - - n, err := io.Copy(pw, src) - if err != nil { - t.Fatal(err) - } - if n != int64(len(data)) { - t.Fatalf("transferred %d, want %d", n, len(data)) - } - if !hook.called { - t.Fatalf("should have called poll.CopyFileRange") - } - pw.Close() - mustContainData(t, pr, data) - }) - t.Run("SrcPipe", func(t *testing.T) { - dst, src, data, hook := newCopyFileRangeTest(t, 255) - src.Close() - - pr, pw, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr.Close() - defer pw.Close() - - // The pipe is empty, and PIPE_BUF is large enough - // for this, by (POSIX) definition, so there is no - // need for an additional goroutine. - if _, err := pw.Write(data); err != nil { - t.Fatal(err) - } - pw.Close() - - n, err := io.Copy(dst, pr) - if err != nil { - t.Fatal(err) - } - if n != int64(len(data)) { - t.Fatalf("transferred %d, want %d", n, len(data)) - } - if !hook.called { - t.Fatalf("should have called poll.CopyFileRange") - } - mustSeekStart(t, dst) - mustContainData(t, dst, data) - }) - }) - t.Run("Nil", func(t *testing.T) { - var nilFile *File - anyFile, err := CreateTemp("", "") - if err != nil { - t.Fatal(err) - } - defer Remove(anyFile.Name()) - defer anyFile.Close() - - if _, err := io.Copy(nilFile, nilFile); err != ErrInvalid { - t.Errorf("io.Copy(nilFile, nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := io.Copy(anyFile, nilFile); err != ErrInvalid { - t.Errorf("io.Copy(anyFile, nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := io.Copy(nilFile, anyFile); err != ErrInvalid { - t.Errorf("io.Copy(nilFile, anyFile) = %v, want %v", err, ErrInvalid) - } - - if _, err := nilFile.ReadFrom(nilFile); err != ErrInvalid { - t.Errorf("nilFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := anyFile.ReadFrom(nilFile); err != ErrInvalid { - t.Errorf("anyFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := nilFile.ReadFrom(anyFile); err != ErrInvalid { - t.Errorf("nilFile.ReadFrom(anyFile) = %v, want %v", err, ErrInvalid) - } - }) -} - func TestSpliceFile(t *testing.T) { sizes := []int{ 1, @@ -476,110 +241,33 @@ func testSpliceToTTY(t *testing.T, proto string, size int64) { } } +var ( + copyFileTests = []copyFileTestFunc{newCopyFileRangeTest} + copyFileHooks = []copyFileTestHook{hookCopyFileRange} +) + +func testCopyFiles(t *testing.T, size, limit int64) { + testCopyFileRange(t, size, limit) +} + func testCopyFileRange(t *testing.T, size int64, limit int64) { - dst, src, data, hook := newCopyFileRangeTest(t, size) - - // If we have a limit, wrap the reader. - var ( - realsrc io.Reader - lr *io.LimitedReader - ) - if limit >= 0 { - lr = &io.LimitedReader{N: limit, R: src} - realsrc = lr - if limit < int64(len(data)) { - data = data[:limit] - } - } else { - realsrc = src - } - - // Now call ReadFrom (through io.Copy), which will hopefully call - // poll.CopyFileRange. - n, err := io.Copy(dst, realsrc) - if err != nil { - t.Fatal(err) - } - - // If we didn't have a limit, we should have called poll.CopyFileRange - // with the right file descriptor arguments. - if limit > 0 && !hook.called { - t.Fatal("never called poll.CopyFileRange") - } - if hook.called && hook.dstfd != int(dst.Fd()) { - t.Fatalf("wrong destination file descriptor: got %d, want %d", hook.dstfd, dst.Fd()) - } - if hook.called && hook.srcfd != int(src.Fd()) { - t.Fatalf("wrong source file descriptor: got %d, want %d", hook.srcfd, src.Fd()) - } - - // Check that the offsets after the transfer make sense, that the size - // of the transfer was reported correctly, and that the destination - // file contains exactly the bytes we expect it to contain. - dstoff, err := dst.Seek(0, io.SeekCurrent) - if err != nil { - t.Fatal(err) - } - srcoff, err := src.Seek(0, io.SeekCurrent) - if err != nil { - t.Fatal(err) - } - if dstoff != srcoff { - t.Errorf("offsets differ: dstoff = %d, srcoff = %d", dstoff, srcoff) - } - if dstoff != int64(len(data)) { - t.Errorf("dstoff = %d, want %d", dstoff, len(data)) - } - if n != int64(len(data)) { - t.Errorf("short ReadFrom: wrote %d bytes, want %d", n, len(data)) - } - mustSeekStart(t, dst) - mustContainData(t, dst, data) - - // If we had a limit, check that it was updated. - if lr != nil { - if want := limit - n; lr.N != want { - t.Fatalf("didn't update limit correctly: got %d, want %d", lr.N, want) - } - } + dst, src, data, hook, name := newCopyFileRangeTest(t, size) + testCopyFile(t, dst, src, data, hook, limit, name) } // newCopyFileRangeTest initializes a new test for copy_file_range. // -// It creates source and destination files, and populates the source file -// with random data of the specified size. It also hooks package os' call -// to poll.CopyFileRange and returns the hook so it can be inspected. -func newCopyFileRangeTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileRangeHook) { +// It hooks package os' call to poll.CopyFileRange and returns the hook, +// so it can be inspected. +func newCopyFileRangeTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileHook, name string) { t.Helper() - hook = hookCopyFileRange(t) - tmp := t.TempDir() + name = "newCopyFileRangeTest" - src, err := Create(filepath.Join(tmp, "src")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { src.Close() }) + dst, src, data = newCopyFileTest(t, size) + hook, _ = hookCopyFileRange(t) - dst, err = Create(filepath.Join(tmp, "dst")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { dst.Close() }) - - // Populate the source file with data, then rewind it, so it can be - // consumed by copy_file_range(2). - prng := rand.New(rand.NewSource(time.Now().Unix())) - data = make([]byte, size) - prng.Read(data) - if _, err := src.Write(data); err != nil { - t.Fatal(err) - } - if _, err := src.Seek(0, io.SeekStart); err != nil { - t.Fatal(err) - } - - return dst, src, data, hook + return } // newSpliceFileTest initializes a new test for splice. @@ -616,63 +304,22 @@ func newSpliceFileTest(t *testing.T, proto string, size int64) (*File, net.Conn, return dst, server, data, hook, func() { <-done } } -// mustContainData ensures that the specified file contains exactly the -// specified data. -func mustContainData(t *testing.T, f *File, data []byte) { - t.Helper() +func hookCopyFileRange(t *testing.T) (hook *copyFileHook, name string) { + name = "hookCopyFileRange" - got := make([]byte, len(data)) - if _, err := io.ReadFull(f, got); err != nil { - t.Fatal(err) - } - if !bytes.Equal(got, data) { - t.Fatalf("didn't get the same data back from %s", f.Name()) - } - if _, err := f.Read(make([]byte, 1)); err != io.EOF { - t.Fatalf("not at EOF") - } -} - -func mustSeekStart(t *testing.T, f *File) { - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatal(err) - } -} - -func hookCopyFileRange(t *testing.T) *copyFileRangeHook { - h := new(copyFileRangeHook) - h.install() - t.Cleanup(h.uninstall) - return h -} - -type copyFileRangeHook struct { - called bool - dstfd int - srcfd int - remain int64 - - written int64 - handled bool - err error - - original func(dst, src *poll.FD, remain int64) (int64, bool, error) -} - -func (h *copyFileRangeHook) install() { - h.original = *PollCopyFileRangeP + hook = new(copyFileHook) + orig := *PollCopyFileRangeP + t.Cleanup(func() { + *PollCopyFileRangeP = orig + }) *PollCopyFileRangeP = func(dst, src *poll.FD, remain int64) (int64, bool, error) { - h.called = true - h.dstfd = dst.Sysfd - h.srcfd = src.Sysfd - h.remain = remain - h.written, h.handled, h.err = h.original(dst, src, remain) - return h.written, h.handled, h.err + hook.called = true + hook.dstfd = dst.Sysfd + hook.srcfd = src.Sysfd + hook.written, hook.handled, hook.err = orig(dst, src, remain) + return hook.written, hook.handled, hook.err } -} - -func (h *copyFileRangeHook) uninstall() { - *PollCopyFileRangeP = h.original + return } func hookSpliceFile(t *testing.T) *spliceFileHook { diff --git a/src/os/readfrom_sendfile_test.go b/src/os/readfrom_sendfile_test.go new file mode 100644 index 00000000..86ef71ee --- /dev/null +++ b/src/os/readfrom_sendfile_test.go @@ -0,0 +1,55 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build solaris + +package os_test + +import ( + "io" + . "os" + "testing" +) + +func BenchmarkSendFile(b *testing.B) { + hook := hookSendFileTB(b) + + // 1 GiB file size for copy. + const fileSize = 1 << 30 + + src, _ := createTempFile(b, "benchmark-sendfile-src", int64(fileSize)) + dst, err := CreateTemp(b.TempDir(), "benchmark-sendfile-dst") + if err != nil { + b.Fatalf("failed to create temporary file of destination: %v", err) + } + b.Cleanup(func() { + dst.Close() + }) + + b.ReportAllocs() + b.SetBytes(int64(fileSize)) + b.ResetTimer() + + for i := 0; i <= b.N; i++ { + sent, err := io.Copy(dst, src) + + if err != nil { + b.Fatalf("failed to copy data: %v", err) + } + if !hook.called { + b.Fatalf("should have called the sendfile(2)") + } + if sent != int64(fileSize) { + b.Fatalf("sent %d bytes, want %d", sent, fileSize) + } + + // Rewind the files for the next iteration. + if _, err := src.Seek(0, io.SeekStart); err != nil { + b.Fatalf("failed to rewind the source file: %v", err) + } + if _, err := dst.Seek(0, io.SeekStart); err != nil { + b.Fatalf("failed to rewind the destination file: %v", err) + } + } +} diff --git a/src/os/readfrom_solaris_test.go b/src/os/readfrom_solaris_test.go new file mode 100644 index 00000000..b11fbef0 --- /dev/null +++ b/src/os/readfrom_solaris_test.go @@ -0,0 +1,60 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "internal/poll" + . "os" + "testing" +) + +var ( + copyFileTests = []copyFileTestFunc{newSendfileTest} + copyFileHooks = []copyFileTestHook{hookSendFile} +) + +func testCopyFiles(t *testing.T, size, limit int64) { + testSendfile(t, size, limit) +} + +func testSendfile(t *testing.T, size int64, limit int64) { + dst, src, data, hook, name := newSendfileTest(t, size) + testCopyFile(t, dst, src, data, hook, limit, name) +} + +// newSendFileTest initializes a new test for sendfile over copy_file_range. +// It hooks package os' call to poll.SendFile and returns the hook, +// so it can be inspected. +func newSendfileTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileHook, name string) { + t.Helper() + + name = "newSendfileTest" + + dst, src, data = newCopyFileTest(t, size) + hook, _ = hookSendFile(t) + + return +} + +func hookSendFile(t *testing.T) (*copyFileHook, string) { + return hookSendFileTB(t), "hookSendFile" +} + +func hookSendFileTB(tb testing.TB) *copyFileHook { + hook := new(copyFileHook) + orig := poll.TestHookDidSendFile + tb.Cleanup(func() { + poll.TestHookDidSendFile = orig + }) + poll.TestHookDidSendFile = func(dstFD *poll.FD, src int, written int64, err error, handled bool) { + hook.called = true + hook.dstfd = dstFD.Sysfd + hook.srcfd = src + hook.written = written + hook.err = err + hook.handled = handled + } + return hook +} diff --git a/src/os/readfrom_unix_test.go b/src/os/readfrom_unix_test.go new file mode 100644 index 00000000..dbe2b683 --- /dev/null +++ b/src/os/readfrom_unix_test.go @@ -0,0 +1,467 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build freebsd || linux || solaris + +package os_test + +import ( + "bytes" + "io" + "math/rand" + . "os" + "runtime" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +type ( + copyFileTestFunc func(*testing.T, int64) (*File, *File, []byte, *copyFileHook, string) + copyFileTestHook func(*testing.T) (*copyFileHook, string) +) + +func TestCopyFile(t *testing.T) { + sizes := []int{ + 1, + 42, + 1025, + syscall.Getpagesize() + 1, + 32769, + } + t.Run("Basic", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), -1) + }) + } + }) + t.Run("Limited", func(t *testing.T) { + t.Run("OneLess", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), int64(size)-1) + }) + } + }) + t.Run("Half", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), int64(size)/2) + }) + } + }) + t.Run("More", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), int64(size)+7) + }) + } + }) + }) + t.Run("DoesntTryInAppendMode", func(t *testing.T) { + for _, newTest := range copyFileTests { + dst, src, data, hook, testName := newTest(t, 42) + + dst2, err := OpenFile(dst.Name(), O_RDWR|O_APPEND, 0755) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer dst2.Close() + + if _, err := io.Copy(dst2, src); err != nil { + t.Fatalf("%s: %v", testName, err) + } + switch runtime.GOOS { + case "illumos", "solaris": // sendfile() on SunOS allows target file with O_APPEND set. + if !hook.called { + t.Fatalf("%s: should have called the hook even with destination in O_APPEND mode", testName) + } + default: + if hook.called { + t.Fatalf("%s: hook shouldn't be called with destination in O_APPEND mode", testName) + } + } + mustSeekStart(t, dst2) + mustContainData(t, dst2, data) // through traditional means + } + }) + t.Run("CopyFileItself", func(t *testing.T) { + for _, hookFunc := range copyFileHooks { + hook, testName := hookFunc(t) + + f, err := CreateTemp("", "file-readfrom-itself-test") + if err != nil { + t.Fatalf("%s: failed to create tmp file: %v", testName, err) + } + t.Cleanup(func() { + f.Close() + Remove(f.Name()) + }) + + data := []byte("hello world!") + if _, err := f.Write(data); err != nil { + t.Fatalf("%s: failed to create and feed the file: %v", testName, err) + } + + if err := f.Sync(); err != nil { + t.Fatalf("%s: failed to save the file: %v", testName, err) + } + + // Rewind it. + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatalf("%s: failed to rewind the file: %v", testName, err) + } + + // Read data from the file itself. + if _, err := io.Copy(f, f); err != nil { + t.Fatalf("%s: failed to read from the file: %v", testName, err) + } + + if hook.written != 0 || hook.handled || hook.err != nil { + t.Fatalf("%s: File.readFrom is expected not to use any zero-copy techniques when copying itself."+ + "got hook.written=%d, hook.handled=%t, hook.err=%v; expected hook.written=0, hook.handled=false, hook.err=nil", + testName, hook.written, hook.handled, hook.err) + } + + switch testName { + case "hookCopyFileRange": + // For copy_file_range(2), it fails and returns EINVAL when the source and target + // refer to the same file and their ranges overlap. The hook should be called to + // get the returned error and fall back to generic copy. + if !hook.called { + t.Fatalf("%s: should have called the hook", testName) + } + case "hookSendFile", "hookSendFileOverCopyFileRange": + // For sendfile(2), it allows the source and target to refer to the same file and overlap. + // The hook should not be called and just fall back to generic copy directly. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook", testName) + } + default: + t.Fatalf("%s: unexpected test", testName) + } + + // Rewind it. + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatalf("%s: failed to rewind the file: %v", testName, err) + } + + data2, err := io.ReadAll(f) + if err != nil { + t.Fatalf("%s: failed to read from the file: %v", testName, err) + } + + // It should wind up a double of the original data. + if s := strings.Repeat(string(data), 2); s != string(data2) { + t.Fatalf("%s: file contained %s, expected %s", testName, data2, s) + } + } + }) + t.Run("NotRegular", func(t *testing.T) { + t.Run("BothPipes", func(t *testing.T) { + for _, hookFunc := range copyFileHooks { + hook, testName := hookFunc(t) + + pr1, pw1, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr1.Close() + defer pw1.Close() + + pr2, pw2, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr2.Close() + defer pw2.Close() + + // The pipe is empty, and PIPE_BUF is large enough + // for this, by (POSIX) definition, so there is no + // need for an additional goroutine. + data := []byte("hello") + if _, err := pw1.Write(data); err != nil { + t.Fatalf("%s: %v", testName, err) + } + pw1.Close() + + n, err := io.Copy(pw2, pr1) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if n != int64(len(data)) { + t.Fatalf("%s: transferred %d, want %d", testName, n, len(data)) + } + switch runtime.GOOS { + case "illumos", "solaris": + // On solaris, We rely on File.Stat to get the size of the source file, + // which doesn't work for pipe. + // On illumos, We skip anything other than regular files conservatively + // for the target file, therefore the hook shouldn't have been called. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook with a source or a destination of pipe", testName) + } + default: + if !hook.called { + t.Fatalf("%s: should have called the hook with both source and destination of pipe", testName) + } + } + pw2.Close() + mustContainData(t, pr2, data) + } + }) + t.Run("DstPipe", func(t *testing.T) { + for _, newTest := range copyFileTests { + dst, src, data, hook, testName := newTest(t, 255) + dst.Close() + + pr, pw, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr.Close() + defer pw.Close() + + n, err := io.Copy(pw, src) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if n != int64(len(data)) { + t.Fatalf("%s: transferred %d, want %d", testName, n, len(data)) + } + switch runtime.GOOS { + case "illumos": + // On illumos, We skip anything other than regular files conservatively + // for the target file, therefore the hook shouldn't have been called. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook with a destination of pipe", testName) + } + default: + if !hook.called { + t.Fatalf("%s: should have called the hook with a destination of pipe", testName) + } + } + pw.Close() + mustContainData(t, pr, data) + } + }) + t.Run("SrcPipe", func(t *testing.T) { + for _, newTest := range copyFileTests { + dst, src, data, hook, testName := newTest(t, 255) + src.Close() + + pr, pw, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr.Close() + defer pw.Close() + + // The pipe is empty, and PIPE_BUF is large enough + // for this, by (POSIX) definition, so there is no + // need for an additional goroutine. + if _, err := pw.Write(data); err != nil { + t.Fatalf("%s: %v", testName, err) + } + pw.Close() + + n, err := io.Copy(dst, pr) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if n != int64(len(data)) { + t.Fatalf("%s: transferred %d, want %d", testName, n, len(data)) + } + switch runtime.GOOS { + case "illumos", "solaris": + // On SunOS, We rely on File.Stat to get the size of the source file, + // which doesn't work for pipe. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook with a source of pipe", testName) + } + default: + if !hook.called { + t.Fatalf("%s: should have called the hook with a source of pipe", testName) + } + } + mustSeekStart(t, dst) + mustContainData(t, dst, data) + } + }) + }) + t.Run("Nil", func(t *testing.T) { + var nilFile *File + anyFile, err := CreateTemp("", "") + if err != nil { + t.Fatal(err) + } + defer Remove(anyFile.Name()) + defer anyFile.Close() + + if _, err := io.Copy(nilFile, nilFile); err != ErrInvalid { + t.Errorf("io.Copy(nilFile, nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := io.Copy(anyFile, nilFile); err != ErrInvalid { + t.Errorf("io.Copy(anyFile, nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := io.Copy(nilFile, anyFile); err != ErrInvalid { + t.Errorf("io.Copy(nilFile, anyFile) = %v, want %v", err, ErrInvalid) + } + + if _, err := nilFile.ReadFrom(nilFile); err != ErrInvalid { + t.Errorf("nilFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := anyFile.ReadFrom(nilFile); err != ErrInvalid { + t.Errorf("anyFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := nilFile.ReadFrom(anyFile); err != ErrInvalid { + t.Errorf("nilFile.ReadFrom(anyFile) = %v, want %v", err, ErrInvalid) + } + }) +} + +func testCopyFile(t *testing.T, dst, src *File, data []byte, hook *copyFileHook, limit int64, testName string) { + // If we have a limit, wrap the reader. + var ( + realsrc io.Reader + lr *io.LimitedReader + ) + if limit >= 0 { + lr = &io.LimitedReader{N: limit, R: src} + realsrc = lr + if limit < int64(len(data)) { + data = data[:limit] + } + } else { + realsrc = src + } + + // Now call ReadFrom (through io.Copy), which will hopefully call + // poll.CopyFileRange or poll.SendFile. + n, err := io.Copy(dst, realsrc) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + + // If we didn't have a limit or had a positive limit, we should have called + // poll.CopyFileRange or poll.SendFile with the right file descriptor arguments. + if limit != 0 && !hook.called { + t.Fatalf("%s: never called the hook", testName) + } + if hook.called && hook.dstfd != int(dst.Fd()) { + t.Fatalf("%s: wrong destination file descriptor: got %d, want %d", testName, hook.dstfd, dst.Fd()) + } + if hook.called && hook.srcfd != int(src.Fd()) { + t.Fatalf("%s: wrong source file descriptor: got %d, want %d", testName, hook.srcfd, src.Fd()) + } + + // Check that the offsets after the transfer make sense, that the size + // of the transfer was reported correctly, and that the destination + // file contains exactly the bytes we expect it to contain. + dstoff, err := dst.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + srcoff, err := src.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if dstoff != srcoff { + t.Errorf("%s: offsets differ: dstoff = %d, srcoff = %d", testName, dstoff, srcoff) + } + if dstoff != int64(len(data)) { + t.Errorf("%s: dstoff = %d, want %d", testName, dstoff, len(data)) + } + if n != int64(len(data)) { + t.Errorf("%s: short ReadFrom: wrote %d bytes, want %d", testName, n, len(data)) + } + mustSeekStart(t, dst) + mustContainData(t, dst, data) + + // If we had a limit, check that it was updated. + if lr != nil { + if want := limit - n; lr.N != want { + t.Fatalf("%s: didn't update limit correctly: got %d, want %d", testName, lr.N, want) + } + } +} + +// mustContainData ensures that the specified file contains exactly the +// specified data. +func mustContainData(t *testing.T, f *File, data []byte) { + t.Helper() + + got := make([]byte, len(data)) + if _, err := io.ReadFull(f, got); err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Fatalf("didn't get the same data back from %s", f.Name()) + } + if _, err := f.Read(make([]byte, 1)); err != io.EOF { + t.Fatalf("not at EOF") + } +} + +func mustSeekStart(t *testing.T, f *File) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } +} + +// newCopyFileTest initializes a new test for copying data between files. +// It creates source and destination files, and populates the source file +// with random data of the specified size, then rewind it, so it can be +// consumed by copy_file_range(2) or sendfile(2). +func newCopyFileTest(t *testing.T, size int64) (dst, src *File, data []byte) { + src, data = createTempFile(t, "test-copy-file-src", size) + + dst, err := CreateTemp(t.TempDir(), "test-copy-file-dst") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { dst.Close() }) + + return +} + +type copyFileHook struct { + called bool + dstfd int + srcfd int + + written int64 + handled bool + err error +} + +func createTempFile(tb testing.TB, name string, size int64) (*File, []byte) { + f, err := CreateTemp(tb.TempDir(), name) + if err != nil { + tb.Fatalf("failed to create temporary file: %v", err) + } + tb.Cleanup(func() { + f.Close() + }) + + randSeed := time.Now().Unix() + tb.Logf("random data seed: %d\n", randSeed) + prng := rand.New(rand.NewSource(randSeed)) + data := make([]byte, size) + prng.Read(data) + if _, err := f.Write(data); err != nil { + tb.Fatalf("failed to create and feed the file: %v", err) + } + if err := f.Sync(); err != nil { + tb.Fatalf("failed to save the file: %v", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + tb.Fatalf("failed to rewind the file: %v", err) + } + + return f, data +} diff --git a/src/os/removeall_at.go b/src/os/removeall_at.go index cc254e00..f52f6213 100644 --- a/src/os/removeall_at.go +++ b/src/os/removeall_at.go @@ -166,20 +166,11 @@ func removeAllFrom(parent *File, base string) error { // we are going to (try to) remove the file. // The contents of this file are not relevant for test caching. func openDirAt(dirfd int, name string) (*File, error) { - var r int - for { - var e error - r, e = unix.Openat(dirfd, name, O_RDONLY|syscall.O_CLOEXEC|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0) - if e == nil { - break - } - - // See comment in openFileNolog. - if e == syscall.EINTR { - continue - } - - return nil, e + r, err := ignoringEINTR2(func() (int, error) { + return unix.Openat(dirfd, name, O_RDONLY|syscall.O_CLOEXEC|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0) + }) + if err != nil { + return nil, err } if !supportsCloseOnExec { diff --git a/src/os/removeall_test.go b/src/os/removeall_test.go index 6aca9802..474db2ca 100644 --- a/src/os/removeall_test.go +++ b/src/os/removeall_test.go @@ -159,40 +159,25 @@ func TestRemoveAllLongPath(t *testing.T) { t.Skip("skipping for not implemented platforms") } - prevDir, err := Getwd() - if err != nil { - t.Fatalf("Could not get wd: %s", err) - } + startPath := t.TempDir() + t.Chdir(startPath) - startPath, err := MkdirTemp("", "TestRemoveAllLongPath-") - if err != nil { - t.Fatalf("Could not create TempDir: %s", err) - } - defer RemoveAll(startPath) - - err = Chdir(startPath) - if err != nil { - t.Fatalf("Could not chdir %s: %s", startPath, err) - } - - // Removing paths with over 4096 chars commonly fails + // Removing paths with over 4096 chars commonly fails. + name := strings.Repeat("a", 100) for i := 0; i < 41; i++ { - name := strings.Repeat("a", 100) - - err = Mkdir(name, 0755) - if err != nil { + if err := Mkdir(name, 0755); err != nil { t.Fatalf("Could not mkdir %s: %s", name, err) } - - err = Chdir(name) - if err != nil { + if err := Chdir(name); err != nil { t.Fatalf("Could not chdir %s: %s", name, err) } } - err = Chdir(prevDir) + // Chdir out of startPath before attempting to remove it, + // otherwise RemoveAll fails on aix, illumos and solaris. + err := Chdir(filepath.Join(startPath, "..")) if err != nil { - t.Fatalf("Could not chdir %s: %s", prevDir, err) + t.Fatalf("Could not chdir: %s", err) } err = RemoveAll(startPath) @@ -202,30 +187,11 @@ func TestRemoveAllLongPath(t *testing.T) { } func TestRemoveAllDot(t *testing.T) { - prevDir, err := Getwd() - if err != nil { - t.Fatalf("Could not get wd: %s", err) - } - tempDir, err := MkdirTemp("", "TestRemoveAllDot-") - if err != nil { - t.Fatalf("Could not create TempDir: %s", err) - } - defer RemoveAll(tempDir) + t.Chdir(t.TempDir()) - err = Chdir(tempDir) - if err != nil { - t.Fatalf("Could not chdir to tempdir: %s", err) - } - - err = RemoveAll(".") - if err == nil { + if err := RemoveAll("."); err == nil { t.Errorf("RemoveAll succeed to remove .") } - - err = Chdir(prevDir) - if err != nil { - t.Fatalf("Could not chdir %s: %s", prevDir, err) - } } func TestRemoveAllDotDot(t *testing.T) { diff --git a/src/os/root.go b/src/os/root.go new file mode 100644 index 00000000..f91c0f75 --- /dev/null +++ b/src/os/root.go @@ -0,0 +1,323 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "errors" + "internal/bytealg" + "internal/stringslite" + "internal/testlog" + "io/fs" + "runtime" + "slices" +) + +// OpenInRoot opens the file name in the directory dir. +// It is equivalent to OpenRoot(dir) followed by opening the file in the root. +// +// OpenInRoot returns an error if any component of the name +// references a location outside of dir. +// +// See [Root] for details and limitations. +func OpenInRoot(dir, name string) (*File, error) { + r, err := OpenRoot(dir) + if err != nil { + return nil, err + } + defer r.Close() + return r.Open(name) +} + +// Root may be used to only access files within a single directory tree. +// +// Methods on Root can only access files and directories beneath a root directory. +// If any component of a file name passed to a method of Root references a location +// outside the root, the method returns an error. +// File names may reference the directory itself (.). +// +// Methods on Root will follow symbolic links, but symbolic links may not +// reference a location outside the root. +// Symbolic links must not be absolute. +// +// Methods on Root do not prohibit traversal of filesystem boundaries, +// Linux bind mounts, /proc special files, or access to Unix device files. +// +// Methods on Root are safe to be used from multiple goroutines simultaneously. +// +// On most platforms, creating a Root opens a file descriptor or handle referencing +// the directory. If the directory is moved, methods on Root reference the original +// directory in its new location. +// +// Root's behavior differs on some platforms: +// +// - When GOOS=windows, file names may not reference Windows reserved device names +// such as NUL and COM1. +// - When GOOS=js, Root is vulnerable to TOCTOU (time-of-check-time-of-use) +// attacks in symlink validation, and cannot ensure that operations will not +// escape the root. +// - When GOOS=plan9 or GOOS=js, Root does not track directories across renames. +// On these platforms, a Root references a directory name, not a file descriptor. +type Root struct { + root *root +} + +const ( + // Maximum number of symbolic links we will follow when resolving a file in a root. + // 8 is __POSIX_SYMLOOP_MAX (the minimum allowed value for SYMLOOP_MAX), + // and a common limit. + rootMaxSymlinks = 8 +) + +// OpenRoot opens the named directory. +// If there is an error, it will be of type *PathError. +func OpenRoot(name string) (*Root, error) { + testlog.Open(name) + return openRootNolog(name) +} + +// Name returns the name of the directory presented to OpenRoot. +// +// It is safe to call Name after [Close]. +func (r *Root) Name() string { + return r.root.Name() +} + +// Close closes the Root. +// After Close is called, methods on Root return errors. +func (r *Root) Close() error { + return r.root.Close() +} + +// Open opens the named file in the root for reading. +// See [Open] for more details. +func (r *Root) Open(name string) (*File, error) { + return r.OpenFile(name, O_RDONLY, 0) +} + +// Create creates or truncates the named file in the root. +// See [Create] for more details. +func (r *Root) Create(name string) (*File, error) { + return r.OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666) +} + +// OpenFile opens the named file in the root. +// See [OpenFile] for more details. +// +// If perm contains bits other than the nine least-significant bits (0o777), +// OpenFile returns an error. +func (r *Root) OpenFile(name string, flag int, perm FileMode) (*File, error) { + if perm&0o777 != perm { + return nil, &PathError{Op: "openat", Path: name, Err: errors.New("unsupported file mode")} + } + r.logOpen(name) + rf, err := rootOpenFileNolog(r, name, flag, perm) + if err != nil { + return nil, err + } + rf.appendMode = flag&O_APPEND != 0 + return rf, nil +} + +// OpenRoot opens the named directory in the root. +// If there is an error, it will be of type *PathError. +func (r *Root) OpenRoot(name string) (*Root, error) { + r.logOpen(name) + return openRootInRoot(r, name) +} + +// Mkdir creates a new directory in the root +// with the specified name and permission bits (before umask). +// See [Mkdir] for more details. +// +// If perm contains bits other than the nine least-significant bits (0o777), +// OpenFile returns an error. +func (r *Root) Mkdir(name string, perm FileMode) error { + if perm&0o777 != perm { + return &PathError{Op: "mkdirat", Path: name, Err: errors.New("unsupported file mode")} + } + return rootMkdir(r, name, perm) +} + +// Remove removes the named file or (empty) directory in the root. +// See [Remove] for more details. +func (r *Root) Remove(name string) error { + return rootRemove(r, name) +} + +// Stat returns a [FileInfo] describing the named file in the root. +// See [Stat] for more details. +func (r *Root) Stat(name string) (FileInfo, error) { + r.logStat(name) + return rootStat(r, name, false) +} + +// Lstat returns a [FileInfo] describing the named file in the root. +// If the file is a symbolic link, the returned FileInfo +// describes the symbolic link. +// See [Lstat] for more details. +func (r *Root) Lstat(name string) (FileInfo, error) { + r.logStat(name) + return rootStat(r, name, true) +} + +func (r *Root) logOpen(name string) { + if log := testlog.Logger(); log != nil { + // This won't be right if r's name has changed since it was opened, + // but it's the best we can do. + log.Open(joinPath(r.Name(), name)) + } +} + +func (r *Root) logStat(name string) { + if log := testlog.Logger(); log != nil { + // This won't be right if r's name has changed since it was opened, + // but it's the best we can do. + log.Stat(joinPath(r.Name(), name)) + } +} + +// splitPathInRoot splits a path into components +// and joins it with the given prefix and suffix. +// +// The path is relative to a Root, and must not be +// absolute, volume-relative, or "". +// +// "." components are removed, except in the last component. +// +// Path separators following the last component are preserved. +func splitPathInRoot(s string, prefix, suffix []string) (_ []string, err error) { + if len(s) == 0 { + return nil, errors.New("empty path") + } + if IsPathSeparator(s[0]) { + return nil, errPathEscapes + } + + if runtime.GOOS == "windows" { + // Windows cleans paths before opening them. + s, err = rootCleanPath(s, prefix, suffix) + if err != nil { + return nil, err + } + prefix = nil + suffix = nil + } + + parts := append([]string{}, prefix...) + i, j := 0, 1 + for { + if j < len(s) && !IsPathSeparator(s[j]) { + // Keep looking for the end of this component. + j++ + continue + } + parts = append(parts, s[i:j]) + // Advance to the next component, or end of the path. + for j < len(s) && IsPathSeparator(s[j]) { + j++ + } + if j == len(s) { + // If this is the last path component, + // preserve any trailing path separators. + parts[len(parts)-1] = s[i:] + break + } + if parts[len(parts)-1] == "." { + // Remove "." components, except at the end. + parts = parts[:len(parts)-1] + } + i = j + } + if len(suffix) > 0 && len(parts) > 0 && parts[len(parts)-1] == "." { + // Remove a trailing "." component if we're joining to a suffix. + parts = parts[:len(parts)-1] + } + parts = append(parts, suffix...) + return parts, nil +} + +// FS returns a file system (an fs.FS) for the tree of files in the root. +// +// The result implements [io/fs.StatFS], [io/fs.ReadFileFS] and +// [io/fs.ReadDirFS]. +func (r *Root) FS() fs.FS { + return (*rootFS)(r) +} + +type rootFS Root + +func (rfs *rootFS) Open(name string) (fs.File, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "open", Path: name, Err: ErrInvalid} + } + f, err := r.Open(name) + if err != nil { + return nil, err + } + return f, nil +} + +func (rfs *rootFS) ReadDir(name string) ([]DirEntry, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "readdir", Path: name, Err: ErrInvalid} + } + + // This isn't efficient: We just open a regular file and ReadDir it. + // Ideally, we would skip creating a *File entirely and operate directly + // on the file descriptor, but that will require some extensive reworking + // of directory reading in general. + // + // This suffices for the moment. + f, err := r.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + dirs, err := f.ReadDir(-1) + slices.SortFunc(dirs, func(a, b DirEntry) int { + return bytealg.CompareString(a.Name(), b.Name()) + }) + return dirs, err +} + +func (rfs *rootFS) ReadFile(name string) ([]byte, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "readfile", Path: name, Err: ErrInvalid} + } + f, err := r.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + return readFileContents(f) +} + +func (rfs *rootFS) Stat(name string) (FileInfo, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "stat", Path: name, Err: ErrInvalid} + } + return r.Stat(name) +} + +// isValidRootFSPath reprots whether name is a valid filename to pass a Root.FS method. +func isValidRootFSPath(name string) bool { + if !fs.ValidPath(name) { + return false + } + if runtime.GOOS == "windows" { + // fs.FS paths are /-separated. + // On Windows, reject the path if it contains any \ separators. + // Other forms of invalid path (for example, "NUL") are handled by + // Root's usual file lookup mechanisms. + if stringslite.IndexByte(name, '\\') >= 0 { + return false + } + } + return true +} diff --git a/src/os/root_js.go b/src/os/root_js.go new file mode 100644 index 00000000..70aa5f9c --- /dev/null +++ b/src/os/root_js.go @@ -0,0 +1,100 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build js && wasm + +package os + +import ( + "errors" + "slices" + "syscall" +) + +// checkPathEscapes reports whether name escapes the root. +// +// Due to the lack of openat, checkPathEscapes is subject to TOCTOU races +// when symlinks change during the resolution process. +func checkPathEscapes(r *Root, name string) error { + return checkPathEscapesInternal(r, name, false) +} + +// checkPathEscapesLstat reports whether name escapes the root. +// It does not resolve symlinks in the final path component. +// +// Due to the lack of openat, checkPathEscapes is subject to TOCTOU races +// when symlinks change during the resolution process. +func checkPathEscapesLstat(r *Root, name string) error { + return checkPathEscapesInternal(r, name, true) +} + +func checkPathEscapesInternal(r *Root, name string, lstat bool) error { + if r.root.closed.Load() { + return ErrClosed + } + parts, err := splitPathInRoot(name, nil, nil) + if err != nil { + return err + } + + i := 0 + symlinks := 0 + base := r.root.name + for i < len(parts) { + if parts[i] == ".." { + // Resolve one or more parent ("..") path components. + end := i + 1 + for end < len(parts) && parts[end] == ".." { + end++ + } + count := end - i + if count > i { + return errPathEscapes + } + parts = slices.Delete(parts, i-count, end) + i -= count + base = r.root.name + for j := range i { + base = joinPath(base, parts[j]) + } + continue + } + + if lstat && i == len(parts)-1 { + break + } + + next := joinPath(base, parts[i]) + fi, err := Lstat(next) + if err != nil { + if IsNotExist(err) { + return nil + } + return underlyingError(err) + } + if fi.Mode()&ModeSymlink != 0 { + link, err := Readlink(next) + if err != nil { + return errPathEscapes + } + symlinks++ + if symlinks > rootMaxSymlinks { + return errors.New("too many symlinks") + } + newparts, err := splitPathInRoot(link, parts[:i], parts[i+1:]) + if err != nil { + return err + } + parts = newparts + continue + } + if !fi.IsDir() && i < len(parts)-1 { + return syscall.ENOTDIR + } + + base = next + i++ + } + return nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/internal/versions/toolchain_go120.go b/src/os/root_nonwindows.go similarity index 56% rename from src/cmd/vendor/golang.org/x/tools/internal/versions/toolchain_go120.go rename to src/os/root_nonwindows.go index 1a9efa12..e40ce4d7 100644 --- a/src/cmd/vendor/golang.org/x/tools/internal/versions/toolchain_go120.go +++ b/src/os/root_nonwindows.go @@ -2,13 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build go1.20 -// +build go1.20 +//go:build !windows -package versions +package os -func init() { - if Compare(toolchain, Go1_20) < 0 { - toolchain = Go1_20 - } +func rootCleanPath(s string, prefix, suffix []string) (string, error) { + return s, nil } diff --git a/src/os/root_noopenat.go b/src/os/root_noopenat.go new file mode 100644 index 00000000..8be55a02 --- /dev/null +++ b/src/os/root_noopenat.go @@ -0,0 +1,116 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (js && wasm) || plan9 + +package os + +import ( + "errors" + "sync/atomic" +) + +// root implementation for platforms with no openat. +// Currently plan9 and js. +type root struct { + name string + closed atomic.Bool +} + +// openRootNolog is OpenRoot. +func openRootNolog(name string) (*Root, error) { + r, err := newRoot(name) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} + } + return r, nil +} + +// openRootInRoot is Root.OpenRoot. +func openRootInRoot(r *Root, name string) (*Root, error) { + if err := checkPathEscapes(r, name); err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + r, err := newRoot(joinPath(r.root.name, name)) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + return r, nil +} + +// newRoot returns a new Root. +// If fd is not a directory, it closes it and returns an error. +func newRoot(name string) (*Root, error) { + fi, err := Stat(name) + if err != nil { + return nil, err.(*PathError).Err + } + if !fi.IsDir() { + return nil, errors.New("not a directory") + } + return &Root{&root{name: name}}, nil +} + +func (r *root) Close() error { + // For consistency with platforms where Root.Close closes a handle, + // mark the Root as closed and return errors from future calls. + r.closed.Store(true) + return nil +} + +func (r *root) Name() string { + return r.name +} + +// rootOpenFileNolog is Root.OpenFile. +func rootOpenFileNolog(r *Root, name string, flag int, perm FileMode) (*File, error) { + if err := checkPathEscapes(r, name); err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + f, err := openFileNolog(joinPath(r.root.name, name), flag, perm) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: underlyingError(err)} + } + return f, nil +} + +func rootStat(r *Root, name string, lstat bool) (FileInfo, error) { + var fi FileInfo + var err error + if lstat { + err = checkPathEscapesLstat(r, name) + if err == nil { + fi, err = Lstat(joinPath(r.root.name, name)) + } + } else { + err = checkPathEscapes(r, name) + if err == nil { + fi, err = Stat(joinPath(r.root.name, name)) + } + } + if err != nil { + return nil, &PathError{Op: "statat", Path: name, Err: underlyingError(err)} + } + return fi, nil +} + +func rootMkdir(r *Root, name string, perm FileMode) error { + if err := checkPathEscapes(r, name); err != nil { + return &PathError{Op: "mkdirat", Path: name, Err: err} + } + if err := Mkdir(joinPath(r.root.name, name), perm); err != nil { + return &PathError{Op: "mkdirat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootRemove(r *Root, name string) error { + if err := checkPathEscapesLstat(r, name); err != nil { + return &PathError{Op: "removeat", Path: name, Err: err} + } + if err := Remove(joinPath(r.root.name, name)); err != nil { + return &PathError{Op: "removeat", Path: name, Err: underlyingError(err)} + } + return nil +} diff --git a/src/os/root_openat.go b/src/os/root_openat.go new file mode 100644 index 00000000..a03208b4 --- /dev/null +++ b/src/os/root_openat.go @@ -0,0 +1,209 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || windows || wasip1 + +package os + +import ( + "runtime" + "slices" + "sync" + "syscall" +) + +// root implementation for platforms with a function to open a file +// relative to a directory. +type root struct { + name string + + // refs is incremented while an operation is using fd. + // closed is set when Close is called. + // fd is closed when closed is true and refs is 0. + mu sync.Mutex + fd sysfdType + refs int // number of active operations + closed bool // set when closed +} + +func (r *root) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + if !r.closed && r.refs == 0 { + syscall.Close(r.fd) + } + r.closed = true + runtime.SetFinalizer(r, nil) // no need for a finalizer any more + return nil +} + +func (r *root) incref() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return ErrClosed + } + r.refs++ + return nil +} + +func (r *root) decref() { + r.mu.Lock() + defer r.mu.Unlock() + if r.refs <= 0 { + panic("bad Root refcount") + } + r.refs-- + if r.closed && r.refs == 0 { + syscall.Close(r.fd) + } +} + +func (r *root) Name() string { + return r.name +} + +func rootMkdir(r *Root, name string, perm FileMode) error { + _, err := doInRoot(r, name, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, mkdirat(parent, name, perm) + }) + if err != nil { + return &PathError{Op: "mkdirat", Path: name, Err: err} + } + return err +} + +func rootRemove(r *Root, name string) error { + _, err := doInRoot(r, name, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, removeat(parent, name) + }) + if err != nil { + return &PathError{Op: "removeat", Path: name, Err: err} + } + return err +} + +// doInRoot performs an operation on a path in a Root. +// +// It opens the directory containing the final element of the path, +// and calls f with the directory FD and name of the final element. +// +// If the path refers to a symlink which should be followed, +// then f must return errSymlink. +// doInRoot will follow the symlink and call f again. +func doInRoot[T any](r *Root, name string, f func(parent sysfdType, name string) (T, error)) (ret T, err error) { + if err := r.root.incref(); err != nil { + return ret, err + } + defer r.root.decref() + + parts, err := splitPathInRoot(name, nil, nil) + if err != nil { + return ret, err + } + + rootfd := r.root.fd + dirfd := rootfd + defer func() { + if dirfd != rootfd { + syscall.Close(dirfd) + } + }() + + // When resolving .. path components, we restart path resolution from the root. + // (We can't openat(dir, "..") to move up to the parent directory, + // because dir may have moved since we opened it.) + // To limit how many opens a malicious path can cause us to perform, we set + // a limit on the total number of path steps and the total number of restarts + // caused by .. components. If *both* limits are exceeded, we halt the operation. + const maxSteps = 255 + const maxRestarts = 8 + + i := 0 + steps := 0 + restarts := 0 + symlinks := 0 + for { + steps++ + if steps > maxSteps && restarts > maxRestarts { + return ret, syscall.ENAMETOOLONG + } + + if parts[i] == ".." { + // Resolve one or more parent ("..") path components. + // + // Rewrite the original path, + // removing the elements eliminated by ".." components, + // and start over from the beginning. + restarts++ + end := i + 1 + for end < len(parts) && parts[end] == ".." { + end++ + } + count := end - i + if count > i { + return ret, errPathEscapes + } + parts = slices.Delete(parts, i-count, end) + i = 0 + if dirfd != rootfd { + syscall.Close(dirfd) + } + dirfd = rootfd + continue + } + + if i == len(parts)-1 { + // This is the last path element. + // Call f to decide what to do with it. + // If f returns errSymlink, this element is a symlink + // which should be followed. + ret, err = f(dirfd, parts[i]) + if _, ok := err.(errSymlink); !ok { + return ret, err + } + } else { + var fd sysfdType + fd, err = rootOpenDir(dirfd, parts[i]) + if err == nil { + if dirfd != rootfd { + syscall.Close(dirfd) + } + dirfd = fd + } else if _, ok := err.(errSymlink); !ok { + return ret, err + } + } + + if e, ok := err.(errSymlink); ok { + symlinks++ + if symlinks > rootMaxSymlinks { + return ret, syscall.ELOOP + } + newparts, err := splitPathInRoot(string(e), parts[:i], parts[i+1:]) + if err != nil { + return ret, err + } + if len(newparts) < i || !slices.Equal(parts[:i], newparts[:i]) { + // Some component in the path which we have already traversed + // has changed. We need to restart parsing from the root. + i = 0 + if dirfd != rootfd { + syscall.Close(dirfd) + } + dirfd = rootfd + } + parts = newparts + continue + } + + i++ + } +} + +// errSymlink reports that a file being operated on is actually a symlink, +// and the target of that symlink. +type errSymlink string + +func (errSymlink) Error() string { panic("errSymlink is not user-visible") } diff --git a/src/os/root_plan9.go b/src/os/root_plan9.go new file mode 100644 index 00000000..08005acc --- /dev/null +++ b/src/os/root_plan9.go @@ -0,0 +1,25 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build plan9 + +package os + +import ( + "internal/filepathlite" +) + +func checkPathEscapes(r *Root, name string) error { + if r.root.closed.Load() { + return ErrClosed + } + if !filepathlite.IsLocal(name) { + return errPathEscapes + } + return nil +} + +func checkPathEscapesLstat(r *Root, name string) error { + return checkPathEscapes(r, name) +} diff --git a/src/os/root_test.go b/src/os/root_test.go new file mode 100644 index 00000000..cbb985b2 --- /dev/null +++ b/src/os/root_test.go @@ -0,0 +1,1200 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "path" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" +) + +// testMaybeRooted calls f in two subtests, +// one with a Root and one with a nil r. +func testMaybeRooted(t *testing.T, f func(t *testing.T, r *os.Root)) { + t.Run("NoRoot", func(t *testing.T) { + t.Chdir(t.TempDir()) + f(t, nil) + }) + t.Run("InRoot", func(t *testing.T) { + t.Chdir(t.TempDir()) + r, err := os.OpenRoot(".") + if err != nil { + t.Fatal(err) + } + defer r.Close() + f(t, r) + }) +} + +// makefs creates a test filesystem layout and returns the path to its root. +// +// Each entry in the slice is a file, directory, or symbolic link to create: +// +// - "d/": directory d +// - "f": file f with contents f +// - "a => b": symlink a with target b +// +// The directory containing the filesystem is always named ROOT. +// $ABS is replaced with the absolute path of the directory containing the filesystem. +// +// Parent directories are automatically created as needed. +// +// makefs calls t.Skip if the layout contains features not supported by the current GOOS. +func makefs(t *testing.T, fs []string) string { + root := path.Join(t.TempDir(), "ROOT") + if err := os.Mkdir(root, 0o777); err != nil { + t.Fatal(err) + } + for _, ent := range fs { + ent = strings.ReplaceAll(ent, "$ABS", root) + base, link, isLink := strings.Cut(ent, " => ") + if isLink { + if runtime.GOOS == "wasip1" && path.IsAbs(link) { + t.Skip("absolute link targets not supported on " + runtime.GOOS) + } + if runtime.GOOS == "plan9" { + t.Skip("symlinks not supported on " + runtime.GOOS) + } + ent = base + } + if err := os.MkdirAll(path.Join(root, path.Dir(base)), 0o777); err != nil { + t.Fatal(err) + } + if isLink { + if err := os.Symlink(link, path.Join(root, base)); err != nil { + t.Fatal(err) + } + } else if strings.HasSuffix(ent, "/") { + if err := os.MkdirAll(path.Join(root, ent), 0o777); err != nil { + t.Fatal(err) + } + } else { + if err := os.WriteFile(path.Join(root, ent), []byte(ent), 0o666); err != nil { + t.Fatal(err) + } + } + } + return root +} + +// A rootTest is a test case for os.Root. +type rootTest struct { + name string + + // fs is the test filesystem layout. See makefs above. + fs []string + + // open is the filename to access in the test. + open string + + // target is the filename that we expect to be accessed, after resolving all symlinks. + // For test cases where the operation fails due to an escaping path such as ../ROOT/x, + // the target is the filename that should not have been opened. + target string + + // ltarget is the filename that we expect to accessed, after resolving all symlinks + // except the last one. This is the file we expect to be removed by Remove or statted + // by Lstat. + // + // If the last path component in open is not a symlink, ltarget should be "". + ltarget string + + // wantError is true if accessing the file should fail. + wantError bool + + // alwaysFails is true if the open operation is expected to fail + // even when using non-openat operations. + // + // This lets us check that tests that are expected to fail because (for example) + // a path escapes the directory root will succeed when the escaping checks are not + // performed. + alwaysFails bool +} + +// run sets up the test filesystem layout, os.OpenDirs the root, and calls f. +func (test *rootTest) run(t *testing.T, f func(t *testing.T, target string, d *os.Root)) { + t.Run(test.name, func(t *testing.T) { + root := makefs(t, test.fs) + d, err := os.OpenRoot(root) + if err != nil { + t.Fatal(err) + } + defer d.Close() + // The target is a file that will be accessed, + // or a file that should not be accessed + // (because doing so escapes the root). + target := test.target + if test.target != "" { + target = filepath.Join(root, test.target) + } + f(t, target, d) + }) +} + +// errEndsTest checks the error result of a test, +// verifying that it succeeded or failed as expected. +// +// It returns true if the test is done due to encountering an expected error. +// false if the test should continue. +func errEndsTest(t *testing.T, err error, wantError bool, format string, args ...any) bool { + t.Helper() + if wantError { + if err == nil { + op := fmt.Sprintf(format, args...) + t.Fatalf("%v = nil; want error", op) + } + return true + } else { + if err != nil { + op := fmt.Sprintf(format, args...) + t.Fatalf("%v = %v; want success", op, err) + } + return false + } +} + +var rootTestCases = []rootTest{{ + name: "plain path", + fs: []string{}, + open: "target", + target: "target", +}, { + name: "path in directory", + fs: []string{ + "a/b/c/", + }, + open: "a/b/c/target", + target: "a/b/c/target", +}, { + name: "symlink", + fs: []string{ + "link => target", + }, + open: "link", + target: "target", + ltarget: "link", +}, { + name: "symlink chain", + fs: []string{ + "link => a/b/c/target", + "a/b => e", + "a/e => ../f", + "f => g/h/i", + "g/h/i => ..", + "g/c/", + }, + open: "link", + target: "g/c/target", + ltarget: "link", +}, { + name: "path with dot", + fs: []string{ + "a/b/", + }, + open: "./a/./b/./target", + target: "a/b/target", +}, { + name: "path with dotdot", + fs: []string{ + "a/b/", + }, + open: "a/../a/b/../../a/b/../b/target", + target: "a/b/target", +}, { + name: "dotdot no symlink", + fs: []string{ + "a/", + }, + open: "a/../target", + target: "target", +}, { + name: "dotdot after symlink", + fs: []string{ + "a => b/c", + "b/c/", + }, + open: "a/../target", + target: func() string { + if runtime.GOOS == "windows" { + // On Windows, the path is cleaned before symlink resolution. + return "target" + } + return "b/target" + }(), +}, { + name: "dotdot before symlink", + fs: []string{ + "a => b/c", + "b/c/", + }, + open: "b/../a/target", + target: "b/c/target", +}, { + name: "symlink ends in dot", + fs: []string{ + "a => b/.", + "b/", + }, + open: "a/target", + target: "b/target", +}, { + name: "directory does not exist", + fs: []string{}, + open: "a/file", + wantError: true, + alwaysFails: true, +}, { + name: "empty path", + fs: []string{}, + open: "", + wantError: true, + alwaysFails: true, +}, { + name: "symlink cycle", + fs: []string{ + "a => a", + }, + open: "a", + ltarget: "a", + wantError: true, + alwaysFails: true, +}, { + name: "path escapes", + fs: []string{}, + open: "../ROOT/target", + target: "target", + wantError: true, +}, { + name: "long path escapes", + fs: []string{ + "a/", + }, + open: "a/../../ROOT/target", + target: "target", + wantError: true, +}, { + name: "absolute symlink", + fs: []string{ + "link => $ABS/target", + }, + open: "link", + ltarget: "link", + target: "target", + wantError: true, +}, { + name: "relative symlink", + fs: []string{ + "link => ../ROOT/target", + }, + open: "link", + target: "target", + ltarget: "link", + wantError: true, +}, { + name: "symlink chain escapes", + fs: []string{ + "link => a/b/c/target", + "a/b => e", + "a/e => ../../ROOT", + "c/", + }, + open: "link", + target: "c/target", + ltarget: "link", + wantError: true, +}} + +func TestRootOpen_File(t *testing.T) { + want := []byte("target") + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.WriteFile(target, want, 0o666); err != nil { + t.Fatal(err) + } + } + f, err := root.Open(test.open) + if errEndsTest(t, err, test.wantError, "root.Open(%q)", test.open) { + return + } + defer f.Close() + got, err := io.ReadAll(f) + if err != nil || !bytes.Equal(got, want) { + t.Errorf(`Dir.Open(%q): read content %q, %v; want %q`, test.open, string(got), err, string(want)) + } + }) + } +} + +func TestRootOpen_Directory(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.Mkdir(target, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target+"/found", nil, 0o666); err != nil { + t.Fatal(err) + } + } + f, err := root.Open(test.open) + if errEndsTest(t, err, test.wantError, "root.Open(%q)", test.open) { + return + } + defer f.Close() + got, err := f.Readdirnames(-1) + if err != nil { + t.Errorf(`Dir.Open(%q).Readdirnames: %v`, test.open, err) + } + if want := []string{"found"}; !slices.Equal(got, want) { + t.Errorf(`Dir.Open(%q).Readdirnames: %q, want %q`, test.open, got, want) + } + }) + } +} + +func TestRootCreate(t *testing.T) { + want := []byte("target") + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + f, err := root.Create(test.open) + if errEndsTest(t, err, test.wantError, "root.Create(%q)", test.open) { + return + } + if _, err := f.Write(want); err != nil { + t.Fatal(err) + } + f.Close() + got, err := os.ReadFile(target) + if err != nil { + t.Fatalf(`reading file created with root.Create(%q): %v`, test.open, err) + } + if !bytes.Equal(got, want) { + t.Fatalf(`reading file created with root.Create(%q): got %q; want %q`, test.open, got, want) + } + }) + } +} + +func TestRootMkdir(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if !wantError { + fi, err := os.Lstat(filepath.Join(root.Name(), test.open)) + if err == nil && fi.Mode().Type() == fs.ModeSymlink { + // This case is trying to mkdir("some symlink"), + // which is an error. + wantError = true + } + } + + err := root.Mkdir(test.open, 0o777) + if errEndsTest(t, err, wantError, "root.Create(%q)", test.open) { + return + } + fi, err := os.Lstat(target) + if err != nil { + t.Fatalf(`stat file created with Root.Mkdir(%q): %v`, test.open, err) + } + if !fi.IsDir() { + t.Fatalf(`stat file created with Root.Mkdir(%q): not a directory`, test.open) + } + }) + } +} + +func TestRootOpenRoot(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.Mkdir(target, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target+"/f", nil, 0o666); err != nil { + t.Fatal(err) + } + } + rr, err := root.OpenRoot(test.open) + if errEndsTest(t, err, test.wantError, "root.OpenRoot(%q)", test.open) { + return + } + defer rr.Close() + f, err := rr.Open("f") + if err != nil { + t.Fatalf(`root.OpenRoot(%q).Open("f") = %v`, test.open, err) + } + f.Close() + }) + } +} + +func TestRootRemoveFile(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if test.ltarget != "" { + // Remove doesn't follow symlinks in the final path component, + // so it will successfully remove ltarget. + wantError = false + target = filepath.Join(root.Name(), test.ltarget) + } else if target != "" { + if err := os.WriteFile(target, nil, 0o666); err != nil { + t.Fatal(err) + } + } + + err := root.Remove(test.open) + if errEndsTest(t, err, wantError, "root.Remove(%q)", test.open) { + return + } + _, err = os.Lstat(target) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf(`stat file removed with Root.Remove(%q): %v, want ErrNotExist`, test.open, err) + } + }) + } +} + +func TestRootRemoveDirectory(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if test.ltarget != "" { + // Remove doesn't follow symlinks in the final path component, + // so it will successfully remove ltarget. + wantError = false + target = filepath.Join(root.Name(), test.ltarget) + } else if target != "" { + if err := os.Mkdir(target, 0o777); err != nil { + t.Fatal(err) + } + } + + err := root.Remove(test.open) + if errEndsTest(t, err, wantError, "root.Remove(%q)", test.open) { + return + } + _, err = os.Lstat(target) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf(`stat file removed with Root.Remove(%q): %v, want ErrNotExist`, test.open, err) + } + }) + } +} + +func TestRootOpenFileAsRoot(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "target") + if err := os.WriteFile(target, nil, 0o666); err != nil { + t.Fatal(err) + } + _, err := os.OpenRoot(target) + if err == nil { + t.Fatal("os.OpenRoot(file) succeeded; want failure") + } + r, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + _, err = r.OpenRoot("target") + if err == nil { + t.Fatal("Root.OpenRoot(file) succeeded; want failure") + } +} + +func TestRootStat(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + const content = "content" + if target != "" { + if err := os.WriteFile(target, []byte(content), 0o666); err != nil { + t.Fatal(err) + } + } + + fi, err := root.Stat(test.open) + if errEndsTest(t, err, test.wantError, "root.Stat(%q)", test.open) { + return + } + if got, want := fi.Name(), filepath.Base(test.open); got != want { + t.Errorf("root.Stat(%q).Name() = %q, want %q", test.open, got, want) + } + if got, want := fi.Size(), int64(len(content)); got != want { + t.Errorf("root.Stat(%q).Size() = %v, want %v", test.open, got, want) + } + }) + } +} + +func TestRootLstat(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + const content = "content" + wantError := test.wantError + if test.ltarget != "" { + // Lstat will stat the final link, rather than following it. + wantError = false + } else if target != "" { + if err := os.WriteFile(target, []byte(content), 0o666); err != nil { + t.Fatal(err) + } + } + + fi, err := root.Lstat(test.open) + if errEndsTest(t, err, wantError, "root.Stat(%q)", test.open) { + return + } + if got, want := fi.Name(), filepath.Base(test.open); got != want { + t.Errorf("root.Stat(%q).Name() = %q, want %q", test.open, got, want) + } + if test.ltarget == "" { + if got := fi.Mode(); got&os.ModeSymlink != 0 { + t.Errorf("root.Stat(%q).Mode() = %v, want non-symlink", test.open, got) + } + if got, want := fi.Size(), int64(len(content)); got != want { + t.Errorf("root.Stat(%q).Size() = %v, want %v", test.open, got, want) + } + } else { + if got := fi.Mode(); got&os.ModeSymlink == 0 { + t.Errorf("root.Stat(%q).Mode() = %v, want symlink", test.open, got) + } + } + }) + } +} + +// A rootConsistencyTest is a test case comparing os.Root behavior with +// the corresponding non-Root function. +// +// These tests verify that, for example, Root.Open("file/./") and os.Open("file/./") +// have the same result, although the specific result may vary by platform. +type rootConsistencyTest struct { + name string + + // fs is the test filesystem layout. See makefs above. + // fsFunc is called to modify the test filesystem, or replace it. + fs []string + fsFunc func(t *testing.T, dir string) string + + // open is the filename to access in the test. + open string + + // detailedErrorMismatch indicates that os.Root and the corresponding non-Root + // function return different errors for this test. + detailedErrorMismatch func(t *testing.T) bool +} + +var rootConsistencyTestCases = []rootConsistencyTest{{ + name: "file", + fs: []string{ + "target", + }, + open: "target", +}, { + name: "dir slash dot", + fs: []string{ + "target/file", + }, + open: "target/.", +}, { + name: "dot", + fs: []string{ + "file", + }, + open: ".", +}, { + name: "file slash dot", + fs: []string{ + "target", + }, + open: "target/.", + detailedErrorMismatch: func(t *testing.T) bool { + // FreeBSD returns EPERM in the non-Root case. + return runtime.GOOS == "freebsd" && strings.HasPrefix(t.Name(), "TestRootConsistencyRemove") + }, +}, { + name: "dir slash", + fs: []string{ + "target/file", + }, + open: "target/", +}, { + name: "dot slash", + fs: []string{ + "file", + }, + open: "./", +}, { + name: "file slash", + fs: []string{ + "target", + }, + open: "target/", + detailedErrorMismatch: func(t *testing.T) bool { + // os.Create returns ENOTDIR or EISDIR depending on the platform. + return runtime.GOOS == "js" + }, +}, { + name: "file in path", + fs: []string{ + "file", + }, + open: "file/target", +}, { + name: "directory in path missing", + open: "dir/target", +}, { + name: "target does not exist", + open: "target", +}, { + name: "symlink slash", + fs: []string{ + "target/file", + "link => target", + }, + open: "link/", +}, { + name: "symlink slash dot", + fs: []string{ + "target/file", + "link => target", + }, + open: "link/.", +}, { + name: "file symlink slash", + fs: []string{ + "target", + "link => target", + }, + open: "link/", + detailedErrorMismatch: func(t *testing.T) bool { + // os.Create returns ENOTDIR or EISDIR depending on the platform. + return runtime.GOOS == "js" + }, +}, { + name: "unresolved symlink", + fs: []string{ + "link => target", + }, + open: "link", +}, { + name: "resolved symlink", + fs: []string{ + "link => target", + "target", + }, + open: "link", +}, { + name: "dotdot in path after symlink", + fs: []string{ + "a => b/c", + "b/c/", + "b/target", + }, + open: "a/../target", +}, { + name: "long file name", + open: strings.Repeat("a", 500), +}, { + name: "unreadable directory", + fs: []string{ + "dir/target", + }, + fsFunc: func(t *testing.T, dir string) string { + os.Chmod(filepath.Join(dir, "dir"), 0) + t.Cleanup(func() { + os.Chmod(filepath.Join(dir, "dir"), 0o700) + }) + return dir + }, + open: "dir/target", +}, { + name: "unix domain socket target", + fsFunc: func(t *testing.T, dir string) string { + return tempDirWithUnixSocket(t, "a") + }, + open: "a", +}, { + name: "unix domain socket in path", + fsFunc: func(t *testing.T, dir string) string { + return tempDirWithUnixSocket(t, "a") + }, + open: "a/b", + detailedErrorMismatch: func(t *testing.T) bool { + // On Windows, os.Root.Open returns "The directory name is invalid." + // and os.Open returns "The file cannot be accessed by the system.". + return runtime.GOOS == "windows" + }, +}, { + name: "question mark", + open: "?", +}, { + name: "nul byte", + open: "\x00", +}} + +func tempDirWithUnixSocket(t *testing.T, name string) string { + dir, err := os.MkdirTemp("", "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }) + addr, err := net.ResolveUnixAddr("unix", filepath.Join(dir, name)) + if err != nil { + t.Skipf("net.ResolveUnixAddr: %v", err) + } + conn, err := net.ListenUnix("unix", addr) + if err != nil { + t.Skipf("net.ListenUnix: %v", err) + } + t.Cleanup(func() { + conn.Close() + }) + return dir +} + +func (test rootConsistencyTest) run(t *testing.T, f func(t *testing.T, path string, r *os.Root) (string, error)) { + if runtime.GOOS == "wasip1" { + // On wasip, non-Root functions clean paths before opening them, + // resulting in inconsistent behavior. + // https://go.dev/issue/69509 + t.Skip("#69509: inconsistent results on wasip1") + } + + t.Run(test.name, func(t *testing.T) { + dir1 := makefs(t, test.fs) + dir2 := makefs(t, test.fs) + if test.fsFunc != nil { + dir1 = test.fsFunc(t, dir1) + dir2 = test.fsFunc(t, dir2) + } + + r, err := os.OpenRoot(dir1) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + res1, err1 := f(t, test.open, r) + res2, err2 := f(t, dir2+"/"+test.open, nil) + + if res1 != res2 || ((err1 == nil) != (err2 == nil)) { + t.Errorf("with root: res=%v", res1) + t.Errorf(" err=%v", err1) + t.Errorf("without root: res=%v", res2) + t.Errorf(" err=%v", err2) + t.Errorf("want consistent results, got mismatch") + } + + if err1 != nil || err2 != nil { + e1, ok := err1.(*os.PathError) + if !ok { + t.Fatalf("with root, expected PathError; got: %v", err1) + } + e2, ok := err2.(*os.PathError) + if !ok { + t.Fatalf("without root, expected PathError; got: %v", err1) + } + detailedErrorMismatch := false + if f := test.detailedErrorMismatch; f != nil { + detailedErrorMismatch = f(t) + } + if runtime.GOOS == "plan9" { + // Plan9 syscall errors aren't comparable. + detailedErrorMismatch = true + } + if !detailedErrorMismatch && e1.Err != e2.Err { + t.Errorf("with root: err=%v", e1.Err) + t.Errorf("without root: err=%v", e2.Err) + t.Errorf("want consistent results, got mismatch") + } + } + }) +} + +func TestRootConsistencyOpen(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var f *os.File + var err error + if r == nil { + f, err = os.Open(path) + } else { + f, err = r.Open(path) + } + if err != nil { + return "", err + } + defer f.Close() + fi, err := f.Stat() + if err == nil && !fi.IsDir() { + b, err := io.ReadAll(f) + return string(b), err + } else { + names, err := f.Readdirnames(-1) + slices.Sort(names) + return fmt.Sprintf("%q", names), err + } + }) + } +} + +func TestRootConsistencyCreate(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var f *os.File + var err error + if r == nil { + f, err = os.Create(path) + } else { + f, err = r.Create(path) + } + if err == nil { + f.Write([]byte("file contents")) + f.Close() + } + return "", err + }) + } +} + +func TestRootConsistencyMkdir(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var err error + if r == nil { + err = os.Mkdir(path, 0o777) + } else { + err = r.Mkdir(path, 0o777) + } + return "", err + }) + } +} + +func TestRootConsistencyRemove(t *testing.T) { + for _, test := range rootConsistencyTestCases { + if test.open == "." || test.open == "./" { + continue // can't remove the root itself + } + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var err error + if r == nil { + err = os.Remove(path) + } else { + err = r.Remove(path) + } + return "", err + }) + } +} + +func TestRootConsistencyStat(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var fi os.FileInfo + var err error + if r == nil { + fi, err = os.Stat(path) + } else { + fi, err = r.Stat(path) + } + if err != nil { + return "", err + } + return fmt.Sprintf("name:%q size:%v mode:%v isdir:%v", fi.Name(), fi.Size(), fi.Mode(), fi.IsDir()), nil + }) + } +} + +func TestRootConsistencyLstat(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var fi os.FileInfo + var err error + if r == nil { + fi, err = os.Lstat(path) + } else { + fi, err = r.Lstat(path) + } + if err != nil { + return "", err + } + return fmt.Sprintf("name:%q size:%v mode:%v isdir:%v", fi.Name(), fi.Size(), fi.Mode(), fi.IsDir()), nil + }) + } +} + +func TestRootRenameAfterOpen(t *testing.T) { + switch runtime.GOOS { + case "windows": + t.Skip("renaming open files not supported on " + runtime.GOOS) + case "js", "plan9": + t.Skip("openat not supported on " + runtime.GOOS) + case "wasip1": + if os.Getenv("GOWASIRUNTIME") == "wazero" { + t.Skip("wazero does not track renamed directories") + } + } + + dir := t.TempDir() + + // Create directory "a" and open it. + if err := os.Mkdir(filepath.Join(dir, "a"), 0o777); err != nil { + t.Fatal(err) + } + dirf, err := os.OpenRoot(filepath.Join(dir, "a")) + if err != nil { + t.Fatal(err) + } + defer dirf.Close() + + // Rename "a" => "b", and create "b/f". + if err := os.Rename(filepath.Join(dir, "a"), filepath.Join(dir, "b")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "b/f"), []byte("hello"), 0o666); err != nil { + t.Fatal(err) + } + + // Open "f", and confirm that we see it. + f, err := dirf.OpenFile("f", os.O_RDONLY, 0) + if err != nil { + t.Fatalf("reading file after renaming parent: %v", err) + } + defer f.Close() + b, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if got, want := string(b), "hello"; got != want { + t.Fatalf("file contents: %q, want %q", got, want) + } + + // f.Name reflects the original path we opened the directory under (".../a"), not "b". + if got, want := f.Name(), dirf.Name()+string(os.PathSeparator)+"f"; got != want { + t.Errorf("f.Name() = %q, want %q", got, want) + } +} + +func TestRootNonPermissionMode(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if _, err := r.OpenFile("file", os.O_RDWR|os.O_CREATE, 0o1777); err == nil { + t.Errorf("r.OpenFile(file, O_RDWR|O_CREATE, 0o1777) succeeded; want error") + } + if err := r.Mkdir("file", 0o1777); err == nil { + t.Errorf("r.Mkdir(file, 0o1777) succeeded; want error") + } +} + +func TestRootUseAfterClose(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + r.Close() + for _, test := range []struct { + name string + f func(r *os.Root, filename string) error + }{{ + name: "Open", + f: func(r *os.Root, filename string) error { + _, err := r.Open(filename) + return err + }, + }, { + name: "Create", + f: func(r *os.Root, filename string) error { + _, err := r.Create(filename) + return err + }, + }, { + name: "OpenFile", + f: func(r *os.Root, filename string) error { + _, err := r.OpenFile(filename, os.O_RDWR, 0o666) + return err + }, + }, { + name: "OpenRoot", + f: func(r *os.Root, filename string) error { + _, err := r.OpenRoot(filename) + return err + }, + }, { + name: "Mkdir", + f: func(r *os.Root, filename string) error { + return r.Mkdir(filename, 0o777) + }, + }} { + err := test.f(r, "target") + pe, ok := err.(*os.PathError) + if !ok || pe.Path != "target" || pe.Err != os.ErrClosed { + t.Errorf(`r.%v = %v; want &PathError{Path: "target", Err: ErrClosed}`, test.name, err) + } + } +} + +func TestRootConcurrentClose(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ch := make(chan error, 1) + go func() { + defer close(ch) + first := true + for { + f, err := r.OpenFile("file", os.O_RDWR|os.O_CREATE, 0o666) + if err != nil { + ch <- err + return + } + if first { + ch <- nil + first = false + } + f.Close() + if runtime.GOARCH == "wasm" { + // TODO(go.dev/issue/71134) can lead to goroutine starvation. + runtime.Gosched() + } + } + }() + if err := <-ch; err != nil { + t.Errorf("OpenFile: %v, want success", err) + } + r.Close() + if err := <-ch; !errors.Is(err, os.ErrClosed) { + t.Errorf("OpenFile: %v, want ErrClosed", err) + } +} + +// TestRootRaceRenameDir attempts to escape a Root by renaming a path component mid-parse. +// +// We create a deeply nested directory: +// +// base/a/a/a/a/ [...] /a +// +// And a path that descends into the tree, then returns to the top using ..: +// +// base/a/a/a/a/ [...] /a/../../../ [..] /../a/f +// +// While opening this file, we rename base/a/a to base/b. +// A naive lookup operation will resolve the path to base/f. +func TestRootRaceRenameDir(t *testing.T) { + dir := t.TempDir() + r, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + const depth = 4 + + os.MkdirAll(dir+"/base/"+strings.Repeat("/a", depth), 0o777) + + path := "base/" + strings.Repeat("a/", depth) + strings.Repeat("../", depth) + "a/f" + os.WriteFile(dir+"/f", []byte("secret"), 0o666) + os.WriteFile(dir+"/base/a/f", []byte("public"), 0o666) + + // Compute how long it takes to open the path in the common case. + const tries = 10 + var total time.Duration + for range tries { + start := time.Now() + f, err := r.Open(path) + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if string(b) != "public" { + t.Fatalf("read %q, want %q", b, "public") + } + f.Close() + total += time.Since(start) + } + avg := total / tries + + // We're trying to exploit a race, so try this a number of times. + for range 100 { + // Start a goroutine to open the file. + gotc := make(chan []byte) + go func() { + f, err := r.Open(path) + if err != nil { + gotc <- nil + } + defer f.Close() + b, _ := io.ReadAll(f) + gotc <- b + }() + + // Wait for the open operation to partially complete, + // and then rename a directory near the root. + time.Sleep(avg / 4) + if err := os.Rename(dir+"/base/a", dir+"/b"); err != nil { + // Windows and Plan9 won't let us rename a directory if we have + // an open handle for it, so an error here is expected. + switch runtime.GOOS { + case "windows", "plan9": + default: + t.Fatal(err) + } + } + + got := <-gotc + os.Rename(dir+"/b", dir+"/base/a") + if len(got) > 0 && string(got) != "public" { + t.Errorf("read file: %q; want error or 'public'", got) + } + } +} + +func TestOpenInRoot(t *testing.T) { + dir := makefs(t, []string{ + "file", + "link => ../ROOT/file", + }) + f, err := os.OpenInRoot(dir, "file") + if err != nil { + t.Fatalf("OpenInRoot(`file`) = %v, want success", err) + } + f.Close() + for _, name := range []string{ + "link", + "../ROOT/file", + dir + "/file", + } { + f, err := os.OpenInRoot(dir, name) + if err == nil { + f.Close() + t.Fatalf("OpenInRoot(%q) = nil, want error", name) + } + } +} diff --git a/src/os/root_unix.go b/src/os/root_unix.go new file mode 100644 index 00000000..02d3b4bd --- /dev/null +++ b/src/os/root_unix.go @@ -0,0 +1,198 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || wasip1 + +package os + +import ( + "errors" + "internal/syscall/unix" + "runtime" + "syscall" +) + +type sysfdType = int + +// openRootNolog is OpenRoot. +func openRootNolog(name string) (*Root, error) { + var fd int + err := ignoringEINTR(func() error { + var err error + fd, _, err = open(name, syscall.O_CLOEXEC, 0) + return err + }) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} + } + return newRoot(fd, name) +} + +// newRoot returns a new Root. +// If fd is not a directory, it closes it and returns an error. +func newRoot(fd int, name string) (*Root, error) { + var fs fileStat + err := ignoringEINTR(func() error { + return syscall.Fstat(fd, &fs.sys) + }) + fillFileStatFromSys(&fs, name) + if err == nil && !fs.IsDir() { + syscall.Close(fd) + return nil, &PathError{Op: "open", Path: name, Err: errors.New("not a directory")} + } + + // There's a race here with fork/exec, which we are + // content to live with. See ../syscall/exec_unix.go. + if !supportsCloseOnExec { + syscall.CloseOnExec(fd) + } + + r := &Root{&root{ + fd: fd, + name: name, + }} + runtime.SetFinalizer(r.root, (*root).Close) + return r, nil +} + +// openRootInRoot is Root.OpenRoot. +func openRootInRoot(r *Root, name string) (*Root, error) { + fd, err := doInRoot(r, name, func(parent int, name string) (fd int, err error) { + ignoringEINTR(func() error { + fd, err = unix.Openat(parent, name, syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) + if isNoFollowErr(err) { + err = checkSymlink(parent, name, err) + } + return err + }) + return fd, err + }) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + return newRoot(fd, name) +} + +// rootOpenFileNolog is Root.OpenFile. +func rootOpenFileNolog(root *Root, name string, flag int, perm FileMode) (*File, error) { + fd, err := doInRoot(root, name, func(parent int, name string) (fd int, err error) { + ignoringEINTR(func() error { + fd, err = unix.Openat(parent, name, syscall.O_NOFOLLOW|syscall.O_CLOEXEC|flag, uint32(perm)) + if isNoFollowErr(err) || err == syscall.ENOTDIR { + err = checkSymlink(parent, name, err) + } + return err + }) + return fd, err + }) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + f := newFile(fd, joinPath(root.Name(), name), kindOpenFile, unix.HasNonblockFlag(flag)) + return f, nil +} + +func rootOpenDir(parent int, name string) (int, error) { + var ( + fd int + err error + ) + ignoringEINTR(func() error { + fd, err = unix.Openat(parent, name, syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_DIRECTORY, 0) + if isNoFollowErr(err) || err == syscall.ENOTDIR { + err = checkSymlink(parent, name, err) + } else if err == syscall.ENOTSUP || err == syscall.EOPNOTSUPP { + // ENOTSUP and EOPNOTSUPP are often, but not always, the same errno. + // Translate both to ENOTDIR, since this indicates a non-terminal + // path component was not a directory. + err = syscall.ENOTDIR + } + return err + }) + return fd, err +} + +func rootStat(r *Root, name string, lstat bool) (FileInfo, error) { + fi, err := doInRoot(r, name, func(parent sysfdType, n string) (FileInfo, error) { + var fs fileStat + if err := unix.Fstatat(parent, n, &fs.sys, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return nil, err + } + fillFileStatFromSys(&fs, name) + if !lstat && fs.Mode()&ModeSymlink != 0 { + return nil, checkSymlink(parent, n, syscall.ELOOP) + } + return &fs, nil + }) + if err != nil { + return nil, &PathError{Op: "statat", Path: name, Err: err} + } + return fi, nil +} + +func mkdirat(fd int, name string, perm FileMode) error { + return ignoringEINTR(func() error { + return unix.Mkdirat(fd, name, syscallMode(perm)) + }) +} + +func removeat(fd int, name string) error { + // The system call interface forces us to know whether + // we are removing a file or directory. Try both. + e := ignoringEINTR(func() error { + return unix.Unlinkat(fd, name, 0) + }) + if e == nil { + return nil + } + e1 := ignoringEINTR(func() error { + return unix.Unlinkat(fd, name, unix.AT_REMOVEDIR) + }) + if e1 == nil { + return nil + } + // Both failed. See comment in Remove for how we decide which error to return. + if e1 != syscall.ENOTDIR { + return e1 + } + return e +} + +// checkSymlink resolves the symlink name in parent, +// and returns errSymlink with the link contents. +// +// If name is not a symlink, return origError. +func checkSymlink(parent int, name string, origError error) error { + link, err := readlinkat(parent, name) + if err != nil { + return origError + } + return errSymlink(link) +} + +func readlinkat(fd int, name string) (string, error) { + for len := 128; ; len *= 2 { + b := make([]byte, len) + var ( + n int + e error + ) + ignoringEINTR(func() error { + n, e = unix.Readlinkat(fd, name, b) + return e + }) + if e == syscall.ERANGE { + continue + } + if e != nil { + return "", e + } + if n < 0 { + n = 0 + } + if n < len { + return string(b[0:n]), nil + } + } +} diff --git a/src/os/root_windows.go b/src/os/root_windows.go new file mode 100644 index 00000000..32dfa070 --- /dev/null +++ b/src/os/root_windows.go @@ -0,0 +1,241 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package os + +import ( + "errors" + "internal/filepathlite" + "internal/stringslite" + "internal/syscall/windows" + "runtime" + "syscall" + "unsafe" +) + +// rootCleanPath uses GetFullPathName to perform lexical path cleaning. +// +// On Windows, file names are lexically cleaned at the start of a file operation. +// For example, on Windows the path `a\..\b` is exactly equivalent to `b` alone, +// even if `a` does not exist or is not a directory. +// +// We use the Windows API function GetFullPathName to perform this cleaning. +// We could do this ourselves, but there are a number of subtle behaviors here, +// and deferring to the OS maintains consistency. +// (For example, `a\.\` cleans to `a\`.) +// +// GetFullPathName operates on absolute paths, and our input path is relative. +// We make the path absolute by prepending a fixed prefix of \\?\?\. +// +// We want to detect paths which use .. components to escape the root. +// We do this by ensuring the cleaned path still begins with \\?\?\. +// We catch the corner case of a path which includes a ..\?\. component +// by rejecting any input paths which contain a ?, which is not a valid character +// in a Windows filename. +func rootCleanPath(s string, prefix, suffix []string) (string, error) { + // Reject paths which include a ? component (see above). + if stringslite.IndexByte(s, '?') >= 0 { + return "", windows.ERROR_INVALID_NAME + } + + const fixedPrefix = `\\?\?` + buf := []byte(fixedPrefix) + for _, p := range prefix { + buf = append(buf, '\\') + buf = append(buf, []byte(p)...) + } + buf = append(buf, '\\') + buf = append(buf, []byte(s)...) + for _, p := range suffix { + buf = append(buf, '\\') + buf = append(buf, []byte(p)...) + } + s = string(buf) + + s, err := syscall.FullPath(s) + if err != nil { + return "", err + } + + s, ok := stringslite.CutPrefix(s, fixedPrefix) + if !ok { + return "", errPathEscapes + } + s = stringslite.TrimPrefix(s, `\`) + if s == "" { + s = "." + } + + if !filepathlite.IsLocal(s) { + return "", errPathEscapes + } + + return s, nil +} + +type sysfdType = syscall.Handle + +// openRootNolog is OpenRoot. +func openRootNolog(name string) (*Root, error) { + if name == "" { + return nil, &PathError{Op: "open", Path: name, Err: syscall.ENOENT} + } + path := fixLongPath(name) + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} + } + return newRoot(fd, name) +} + +// newRoot returns a new Root. +// If fd is not a directory, it closes it and returns an error. +func newRoot(fd syscall.Handle, name string) (*Root, error) { + // Check that this is a directory. + // + // If we get any errors here, ignore them; worst case we create a Root + // which returns errors when you try to use it. + var fi syscall.ByHandleFileInformation + err := syscall.GetFileInformationByHandle(fd, &fi) + if err == nil && fi.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY == 0 { + syscall.CloseHandle(fd) + return nil, &PathError{Op: "open", Path: name, Err: errors.New("not a directory")} + } + + r := &Root{&root{ + fd: fd, + name: name, + }} + runtime.SetFinalizer(r.root, (*root).Close) + return r, nil +} + +// openRootInRoot is Root.OpenRoot. +func openRootInRoot(r *Root, name string) (*Root, error) { + fd, err := doInRoot(r, name, rootOpenDir) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + return newRoot(fd, name) +} + +// rootOpenFileNolog is Root.OpenFile. +func rootOpenFileNolog(root *Root, name string, flag int, perm FileMode) (*File, error) { + fd, err := doInRoot(root, name, func(parent syscall.Handle, name string) (syscall.Handle, error) { + return openat(parent, name, flag, perm) + }) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + return newFile(fd, joinPath(root.Name(), name), "file"), nil +} + +func openat(dirfd syscall.Handle, name string, flag int, perm FileMode) (syscall.Handle, error) { + h, err := windows.Openat(dirfd, name, flag|syscall.O_CLOEXEC|windows.O_NOFOLLOW_ANY, syscallMode(perm)) + if err == syscall.ELOOP || err == syscall.ENOTDIR { + if link, err := readReparseLinkAt(dirfd, name); err == nil { + return syscall.InvalidHandle, errSymlink(link) + } + } + return h, err +} + +func readReparseLinkAt(dirfd syscall.Handle, name string) (string, error) { + objectName, err := windows.NewNTUnicodeString(name) + if err != nil { + return "", err + } + objAttrs := &windows.OBJECT_ATTRIBUTES{ + ObjectName: objectName, + } + if dirfd != syscall.InvalidHandle { + objAttrs.RootDirectory = dirfd + } + objAttrs.Length = uint32(unsafe.Sizeof(*objAttrs)) + var h syscall.Handle + err = windows.NtCreateFile( + &h, + windows.FILE_GENERIC_READ, + objAttrs, + &windows.IO_STATUS_BLOCK{}, + nil, + uint32(syscall.FILE_ATTRIBUTE_NORMAL), + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + windows.FILE_OPEN, + windows.FILE_SYNCHRONOUS_IO_NONALERT|windows.FILE_OPEN_REPARSE_POINT, + 0, + 0, + ) + if err != nil { + return "", err + } + defer syscall.CloseHandle(h) + return readReparseLinkHandle(h) +} + +func rootOpenDir(parent syscall.Handle, name string) (syscall.Handle, error) { + h, err := openat(parent, name, syscall.O_RDONLY|syscall.O_CLOEXEC|windows.O_DIRECTORY, 0) + if err == syscall.ERROR_FILE_NOT_FOUND { + // Windows returns: + // - ERROR_PATH_NOT_FOUND if any path compoenent before the leaf + // does not exist or is not a directory. + // - ERROR_FILE_NOT_FOUND if the leaf does not exist. + // + // This differs from Unix behavior, which is: + // - ENOENT if any path component does not exist, including the leaf. + // - ENOTDIR if any path component before the leaf is not a directory. + // + // We map syscall.ENOENT to ERROR_FILE_NOT_FOUND and syscall.ENOTDIR + // to ERROR_PATH_NOT_FOUND, but the Windows errors don't quite match. + // + // For consistency with os.Open, convert ERROR_FILE_NOT_FOUND here into + // ERROR_PATH_NOT_FOUND, since we're opening a non-leaf path component. + err = syscall.ERROR_PATH_NOT_FOUND + } + return h, err +} + +func rootStat(r *Root, name string, lstat bool) (FileInfo, error) { + if len(name) > 0 && IsPathSeparator(name[len(name)-1]) { + // When a filename ends with a path separator, + // Lstat behaves like Stat. + // + // This behavior is not based on a principled decision here, + // merely the empirical evidence that Lstat behaves this way. + lstat = false + } + fi, err := doInRoot(r, name, func(parent syscall.Handle, n string) (FileInfo, error) { + fd, err := openat(parent, n, windows.O_OPEN_REPARSE, 0) + if err != nil { + return nil, err + } + defer syscall.CloseHandle(fd) + fi, err := statHandle(name, fd) + if err != nil { + return nil, err + } + if !lstat && fi.(*fileStat).isReparseTagNameSurrogate() { + link, err := readReparseLinkHandle(fd) + if err != nil { + return nil, err + } + return nil, errSymlink(link) + } + return fi, nil + }) + if err != nil { + return nil, &PathError{Op: "statat", Path: name, Err: err} + } + return fi, nil +} + +func mkdirat(dirfd syscall.Handle, name string, perm FileMode) error { + return windows.Mkdirat(dirfd, name, syscallMode(perm)) +} + +func removeat(dirfd syscall.Handle, name string) error { + return windows.Deleteat(dirfd, name) +} diff --git a/src/os/root_windows_test.go b/src/os/root_windows_test.go new file mode 100644 index 00000000..62e20971 --- /dev/null +++ b/src/os/root_windows_test.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package os_test + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +// Verify that Root.Open rejects Windows reserved names. +func TestRootWindowsDeviceNames(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if f, err := r.Open("NUL"); err == nil { + t.Errorf(`r.Open("NUL") succeeded; want error"`) + f.Close() + } +} + +// Verify that Root.Open is case-insensitive. +// (The wrong options to NtOpenFile could make operations case-sensitive, +// so this is worth checking.) +func TestRootWindowsCaseInsensitivity(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "file"), nil, 0666); err != nil { + t.Fatal(err) + } + r, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + f, err := r.Open("FILE") + if err != nil { + t.Fatal(err) + } + f.Close() + if err := r.Remove("FILE"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "file")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("os.Stat(file) after deletion: %v, want ErrNotFound", err) + } +} diff --git a/src/os/signal/signal.go b/src/os/signal/signal.go index 9a4cd64f..b9fe16ba 100644 --- a/src/os/signal/signal.go +++ b/src/os/signal/signal.go @@ -7,6 +7,7 @@ package signal import ( "context" "os" + "slices" "sync" ) @@ -217,7 +218,7 @@ func Stop(c chan<- os.Signal) { for i, s := range handlers.stopping { if s.c == c { - handlers.stopping = append(handlers.stopping[:i], handlers.stopping[i+1:]...) + handlers.stopping = slices.Delete(handlers.stopping, i, i+1) break } } diff --git a/src/os/stat_wasip1.go b/src/os/stat_wasip1.go index 85a36498..dcf2e38d 100644 --- a/src/os/stat_wasip1.go +++ b/src/os/stat_wasip1.go @@ -15,7 +15,6 @@ import ( func fillFileStatFromSys(fs *fileStat, name string) { fs.name = filepathlite.Base(name) fs.size = int64(fs.sys.Size) - fs.mode = FileMode(fs.sys.Mode) fs.modTime = time.Unix(0, int64(fs.sys.Mtime)) switch fs.sys.Filetype { @@ -32,6 +31,15 @@ func fillFileStatFromSys(fs *fileStat, name string) { case syscall.FILETYPE_SYMBOLIC_LINK: fs.mode |= ModeSymlink } + + // WASI does not support unix-like permissions, but Go programs are likely + // to expect the permission bits to not be zero so we set defaults to help + // avoid breaking applications that are migrating to WASM. + if fs.sys.Filetype == syscall.FILETYPE_DIRECTORY { + fs.mode |= 0700 + } else { + fs.mode |= 0600 + } } // For testing. diff --git a/src/os/tempfile_test.go b/src/os/tempfile_test.go index 82f0aabd..f2b4ffa7 100644 --- a/src/os/tempfile_test.go +++ b/src/os/tempfile_test.go @@ -17,13 +17,7 @@ import ( func TestCreateTemp(t *testing.T) { t.Parallel() - dir, err := MkdirTemp("", "TestCreateTempBadDir") - if err != nil { - t.Fatal(err) - } - defer RemoveAll(dir) - - nonexistentDir := filepath.Join(dir, "_not_exists_") + nonexistentDir := filepath.Join(t.TempDir(), "_not_exists_") f, err := CreateTemp(nonexistentDir, "foo") if f != nil || err == nil { t.Errorf("CreateTemp(%q, `foo`) = %v, %v", nonexistentDir, f, err) @@ -57,11 +51,7 @@ func TestCreateTempPattern(t *testing.T) { func TestCreateTempBadPattern(t *testing.T) { t.Parallel() - tmpDir, err := MkdirTemp("", t.Name()) - if err != nil { - t.Fatal(err) - } - defer RemoveAll(tmpDir) + tmpDir := t.TempDir() const sep = string(PathSeparator) tests := []struct { @@ -152,14 +142,8 @@ func TestMkdirTemp(t *testing.T) { func TestMkdirTempBadDir(t *testing.T) { t.Parallel() - dir, err := MkdirTemp("", "MkdirTempBadDir") - if err != nil { - t.Fatal(err) - } - defer RemoveAll(dir) - - badDir := filepath.Join(dir, "not-exist") - _, err = MkdirTemp(badDir, "foo") + badDir := filepath.Join(t.TempDir(), "not-exist") + _, err := MkdirTemp(badDir, "foo") if pe, ok := err.(*fs.PathError); !ok || !IsNotExist(err) || pe.Path != badDir { t.Errorf("TempDir error = %#v; want PathError for path %q satisfying IsNotExist", err, badDir) } @@ -168,11 +152,7 @@ func TestMkdirTempBadDir(t *testing.T) { func TestMkdirTempBadPattern(t *testing.T) { t.Parallel() - tmpDir, err := MkdirTemp("", t.Name()) - if err != nil { - t.Fatal(err) - } - defer RemoveAll(tmpDir) + tmpDir := t.TempDir() const sep = string(PathSeparator) tests := []struct { diff --git a/src/os/testdata/dirfs/dir/x b/src/os/testdata/dirfs/dir/x old mode 100644 new mode 100755 diff --git a/src/os/user/lookup_windows.go b/src/os/user/lookup_windows.go index a48fc897..e0e77f3e 100644 --- a/src/os/user/lookup_windows.go +++ b/src/os/user/lookup_windows.go @@ -5,9 +5,11 @@ package user import ( + "errors" "fmt" "internal/syscall/windows" "internal/syscall/windows/registry" + "runtime" "syscall" "unsafe" ) @@ -84,16 +86,73 @@ func getProfilesDirectory() (string, error) { } } +func isServiceAccount(sid *syscall.SID) bool { + if !windows.IsValidSid(sid) { + // We don't accept SIDs from the public API, so this should never happen. + // Better be on the safe side and validate anyway. + return false + } + // The following RIDs are considered service user accounts as per + // https://learn.microsoft.com/en-us/windows/win32/secauthz/well-known-sids and + // https://learn.microsoft.com/en-us/windows/win32/services/service-user-accounts: + // - "S-1-5-18": LocalSystem + // - "S-1-5-19": LocalService + // - "S-1-5-20": NetworkService + if windows.GetSidSubAuthorityCount(sid) != windows.SID_REVISION || + windows.GetSidIdentifierAuthority(sid) != windows.SECURITY_NT_AUTHORITY { + return false + } + switch windows.GetSidSubAuthority(sid, 0) { + case windows.SECURITY_LOCAL_SYSTEM_RID, + windows.SECURITY_LOCAL_SERVICE_RID, + windows.SECURITY_NETWORK_SERVICE_RID: + return true + } + return false +} + +func isValidUserAccountType(sid *syscall.SID, sidType uint32) bool { + switch sidType { + case syscall.SidTypeUser: + return true + case syscall.SidTypeWellKnownGroup: + return isServiceAccount(sid) + } + return false +} + +func isValidGroupAccountType(sidType uint32) bool { + switch sidType { + case syscall.SidTypeGroup: + return true + case syscall.SidTypeWellKnownGroup: + // Some well-known groups are also considered service accounts, + // so isValidUserAccountType would return true for them. + // We have historically allowed them in LookupGroup and LookupGroupId, + // so don't treat them as invalid here. + return true + case syscall.SidTypeAlias: + // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/7b2aeb27-92fc-41f6-8437-deb65d950921#gt_0387e636-5654-4910-9519-1f8326cf5ec0 + // SidTypeAlias should also be treated as a group type next to SidTypeGroup + // and SidTypeWellKnownGroup: + // "alias object -> resource group: A group object..." + // + // Tests show that "Administrators" can be considered of type SidTypeAlias. + return true + } + return false +} + // lookupUsernameAndDomain obtains the username and domain for usid. -func lookupUsernameAndDomain(usid *syscall.SID) (username, domain string, e error) { - username, domain, t, e := usid.LookupAccount("") +func lookupUsernameAndDomain(usid *syscall.SID) (username, domain string, sidType uint32, e error) { + username, domain, sidType, e = usid.LookupAccount("") if e != nil { - return "", "", e + return "", "", 0, e } - if t != syscall.SidTypeUser { - return "", "", fmt.Errorf("user: should be user account type, not %d", t) + if !isValidUserAccountType(usid, sidType) { + return "", "", 0, fmt.Errorf("user: should be user account type, not %d", sidType) } - return username, domain, nil + return username, domain, sidType, nil } // findHomeDirInRegistry finds the user home path based on the uid. @@ -116,13 +175,7 @@ func lookupGroupName(groupname string) (string, error) { if e != nil { return "", e } - // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/7b2aeb27-92fc-41f6-8437-deb65d950921#gt_0387e636-5654-4910-9519-1f8326cf5ec0 - // SidTypeAlias should also be treated as a group type next to SidTypeGroup - // and SidTypeWellKnownGroup: - // "alias object -> resource group: A group object..." - // - // Tests show that "Administrators" can be considered of type SidTypeAlias. - if t != syscall.SidTypeGroup && t != syscall.SidTypeWellKnownGroup && t != syscall.SidTypeAlias { + if !isValidGroupAccountType(t) { return "", fmt.Errorf("lookupGroupName: should be group account type, not %d", t) } return sid.String() @@ -149,17 +202,13 @@ func listGroupsForUsernameAndDomain(username, domain string) ([]string, error) { // NetUserGetLocalGroups() would return a list of LocalGroupUserInfo0 // elements which hold the names of local groups where the user participates. // The list does not follow any sorting order. - // - // If no groups can be found for this user, NetUserGetLocalGroups() should - // always return the SID of a single group called "None", which - // also happens to be the primary group for the local user. err = windows.NetUserGetLocalGroups(nil, q, 0, windows.LG_INCLUDE_INDIRECT, &p0, windows.MAX_PREFERRED_LENGTH, &entriesRead, &totalEntries) if err != nil { return nil, err } defer syscall.NetApiBufferFree(p0) if entriesRead == 0 { - return nil, fmt.Errorf("listGroupsForUsernameAndDomain: NetUserGetLocalGroups() returned an empty list for domain: %s, username: %s", domain, username) + return nil, nil } entries := (*[1024]windows.LocalGroupUserInfo0)(unsafe.Pointer(p0))[:entriesRead:entriesRead] var sids []string @@ -200,36 +249,103 @@ var ( ) func current() (*User, error) { - t, e := syscall.OpenCurrentProcessToken() - if e != nil { - return nil, e + // Use runAsProcessOwner to ensure that we can access the process token + // when calling syscall.OpenCurrentProcessToken if the current thread + // is impersonating a different user. See https://go.dev/issue/68647. + var usr *User + err := runAsProcessOwner(func() error { + t, e := syscall.OpenCurrentProcessToken() + if e != nil { + return e + } + defer t.Close() + u, e := t.GetTokenUser() + if e != nil { + return e + } + pg, e := t.GetTokenPrimaryGroup() + if e != nil { + return e + } + uid, e := u.User.Sid.String() + if e != nil { + return e + } + gid, e := pg.PrimaryGroup.String() + if e != nil { + return e + } + dir, e := t.GetUserProfileDirectory() + if e != nil { + return e + } + username, e := windows.GetUserName(syscall.NameSamCompatible) + if e != nil { + return e + } + displayName, e := windows.GetUserName(syscall.NameDisplay) + if e != nil { + // Historically, the username is used as fallback + // when the display name can't be retrieved. + displayName = username + } + usr = &User{ + Uid: uid, + Gid: gid, + Username: username, + Name: displayName, + HomeDir: dir, + } + return nil + }) + return usr, err +} + +// runAsProcessOwner runs f in the context of the current process owner, +// that is, removing any impersonation that may be in effect before calling f, +// and restoring the impersonation afterwards. +func runAsProcessOwner(f func() error) error { + var impersonationRollbackErr error + runtime.LockOSThread() + defer func() { + // If impersonation failed, the thread is running with the wrong token, + // so it's better to terminate it. + // This is achieved by not calling runtime.UnlockOSThread. + if impersonationRollbackErr != nil { + println("os/user: failed to revert to previous token:", impersonationRollbackErr.Error()) + runtime.Goexit() + } else { + runtime.UnlockOSThread() + } + }() + prevToken, isProcessToken, err := getCurrentToken() + if err != nil { + return fmt.Errorf("os/user: failed to get current token: %w", err) } - defer t.Close() - u, e := t.GetTokenUser() - if e != nil { - return nil, e + defer prevToken.Close() + if !isProcessToken { + if err = windows.RevertToSelf(); err != nil { + return fmt.Errorf("os/user: failed to revert to self: %w", err) + } + defer func() { + impersonationRollbackErr = windows.ImpersonateLoggedOnUser(prevToken) + }() } - pg, e := t.GetTokenPrimaryGroup() - if e != nil { - return nil, e + return f() +} + +// getCurrentToken returns the current thread token, or +// the process token if the thread doesn't have a token. +func getCurrentToken() (t syscall.Token, isProcessToken bool, err error) { + thread, _ := windows.GetCurrentThread() + // Need TOKEN_DUPLICATE and TOKEN_IMPERSONATE to use the token in ImpersonateLoggedOnUser. + err = windows.OpenThreadToken(thread, syscall.TOKEN_QUERY|syscall.TOKEN_DUPLICATE|syscall.TOKEN_IMPERSONATE, true, &t) + if errors.Is(err, windows.ERROR_NO_TOKEN) { + // Not impersonating, use the process token. + isProcessToken = true + t, err = syscall.OpenCurrentProcessToken() } - uid, e := u.User.Sid.String() - if e != nil { - return nil, e - } - gid, e := pg.PrimaryGroup.String() - if e != nil { - return nil, e - } - dir, e := t.GetUserProfileDirectory() - if e != nil { - return nil, e - } - username, domain, e := lookupUsernameAndDomain(u.User.Sid) - if e != nil { - return nil, e - } - return newUser(uid, gid, dir, username, domain) + return t, isProcessToken, err } // lookupUserPrimaryGroup obtains the primary group SID for a user using this method: @@ -290,11 +406,7 @@ func lookupUserPrimaryGroup(username, domain string) (string, error) { } func newUserFromSid(usid *syscall.SID) (*User, error) { - username, domain, e := lookupUsernameAndDomain(usid) - if e != nil { - return nil, e - } - gid, e := lookupUserPrimaryGroup(username, domain) + username, domain, sidType, e := lookupUsernameAndDomain(usid) if e != nil { return nil, e } @@ -302,6 +414,19 @@ func newUserFromSid(usid *syscall.SID) (*User, error) { if e != nil { return nil, e } + var gid string + if sidType == syscall.SidTypeWellKnownGroup { + // The SID does not contain a domain; this function's domain variable has + // been populated with the SID's identifier authority. This happens with + // special service user accounts such as "NT AUTHORITY\LocalSystem". + // In this case, gid is the same as the user SID. + gid = uid + } else { + gid, e = lookupUserPrimaryGroup(username, domain) + if e != nil { + return nil, e + } + } // If this user has logged in at least once their home path should be stored // in the registry under the specified SID. References: // https://social.technet.microsoft.com/wiki/contents/articles/13895.how-to-remove-a-corrupted-user-profile-from-the-registry.aspx @@ -331,7 +456,7 @@ func lookupUser(username string) (*User, error) { if e != nil { return nil, e } - if t != syscall.SidTypeUser { + if !isValidUserAccountType(sid, t) { return nil, fmt.Errorf("user: should be user account type, not %d", t) } return newUserFromSid(sid) @@ -362,24 +487,52 @@ func lookupGroupId(gid string) (*Group, error) { if err != nil { return nil, err } - if t != syscall.SidTypeGroup && t != syscall.SidTypeWellKnownGroup && t != syscall.SidTypeAlias { + if !isValidGroupAccountType(t) { return nil, fmt.Errorf("lookupGroupId: should be group account type, not %d", t) } return &Group{Name: groupname, Gid: gid}, nil } func listGroups(user *User) ([]string, error) { - sid, err := syscall.StringToSid(user.Uid) - if err != nil { - return nil, err - } - username, domain, err := lookupUsernameAndDomain(sid) - if err != nil { - return nil, err - } - sids, err := listGroupsForUsernameAndDomain(username, domain) - if err != nil { - return nil, err + var sids []string + if u, err := Current(); err == nil && u.Uid == user.Uid { + // It is faster and more reliable to get the groups + // of the current user from the current process token. + err := runAsProcessOwner(func() error { + t, err := syscall.OpenCurrentProcessToken() + if err != nil { + return err + } + defer t.Close() + groups, err := windows.GetTokenGroups(t) + if err != nil { + return err + } + for _, g := range groups.AllGroups() { + sid, err := g.Sid.String() + if err != nil { + return err + } + sids = append(sids, sid) + } + return nil + }) + if err != nil { + return nil, err + } + } else { + sid, err := syscall.StringToSid(user.Uid) + if err != nil { + return nil, err + } + username, domain, _, err := lookupUsernameAndDomain(sid) + if err != nil { + return nil, err + } + sids, err = listGroupsForUsernameAndDomain(username, domain) + if err != nil { + return nil, err + } } // Add the primary group of the user to the list if it is not already there. // This is done only to comply with the POSIX concept of a primary group. diff --git a/src/os/user/user_test.go b/src/os/user/user_test.go index fa597b78..31486aed 100644 --- a/src/os/user/user_test.go +++ b/src/os/user/user_test.go @@ -45,8 +45,9 @@ func TestCurrent(t *testing.T) { } func BenchmarkCurrent(b *testing.B) { + // Benchmark current instead of Current because Current caches the result. for i := 0; i < b.N; i++ { - Current() + current() } } diff --git a/src/os/user/user_windows_test.go b/src/os/user/user_windows_test.go new file mode 100644 index 00000000..c7150337 --- /dev/null +++ b/src/os/user/user_windows_test.go @@ -0,0 +1,279 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package user + +import ( + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "internal/syscall/windows" + "internal/testenv" + "os" + "os/exec" + "runtime" + "strconv" + "syscall" + "testing" + "unsafe" +) + +// windowsTestAccount creates a test user and returns a token for that user. +// If the user already exists, it will be deleted and recreated. +// The caller is responsible for closing the token. +func windowsTestAccount(t *testing.T) (syscall.Token, *User) { + if testenv.Builder() == "" { + // Adding and deleting users requires special permissions. + // Even if we have them, we don't want to create users on + // on dev machines, as they may not be cleaned up. + // See https://dev.go/issue/70396. + t.Skip("skipping non-hermetic test outside of Go builders") + } + const testUserName = "GoStdTestUser01" + var password [33]byte + rand.Read(password[:]) + // Add special chars to ensure it satisfies password requirements. + pwd := base64.StdEncoding.EncodeToString(password[:]) + "_-As@!%*(1)4#2" + name, err := syscall.UTF16PtrFromString(testUserName) + if err != nil { + t.Fatal(err) + } + pwd16, err := syscall.UTF16PtrFromString(pwd) + if err != nil { + t.Fatal(err) + } + userInfo := windows.UserInfo1{ + Name: name, + Password: pwd16, + Priv: windows.USER_PRIV_USER, + } + // Create user. + err = windows.NetUserAdd(nil, 1, (*byte)(unsafe.Pointer(&userInfo)), nil) + if errors.Is(err, syscall.ERROR_ACCESS_DENIED) { + t.Skip("skipping test; don't have permission to create user") + } + if errors.Is(err, windows.NERR_UserExists) { + // User already exists, delete and recreate. + if err = windows.NetUserDel(nil, name); err != nil { + t.Fatal(err) + } + if err = windows.NetUserAdd(nil, 1, (*byte)(unsafe.Pointer(&userInfo)), nil); err != nil { + t.Fatal(err) + } + } else if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err = windows.NetUserDel(nil, name); err != nil { + if !errors.Is(err, windows.NERR_UserNotFound) { + t.Fatal(err) + } + } + }) + domain, err := syscall.UTF16PtrFromString(".") + if err != nil { + t.Fatal(err) + } + const LOGON32_PROVIDER_DEFAULT = 0 + const LOGON32_LOGON_INTERACTIVE = 2 + var token syscall.Token + if err = windows.LogonUser(name, domain, pwd16, LOGON32_LOGON_INTERACTIVE, LOGON32_PROVIDER_DEFAULT, &token); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + token.Close() + }) + usr, err := Lookup(testUserName) + if err != nil { + t.Fatal(err) + } + return token, usr +} + +func TestImpersonatedSelf(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + want, err := current() + if err != nil { + t.Fatal(err) + } + + levels := []uint32{ + windows.SecurityAnonymous, + windows.SecurityIdentification, + windows.SecurityImpersonation, + windows.SecurityDelegation, + } + for _, level := range levels { + t.Run(strconv.Itoa(int(level)), func(t *testing.T) { + if err = windows.ImpersonateSelf(level); err != nil { + t.Fatal(err) + } + defer windows.RevertToSelf() + + got, err := current() + if level == windows.SecurityAnonymous { + // We can't get the process token when using an anonymous token, + // so we expect an error here. + if err == nil { + t.Fatal("expected error") + } + return + } + if err != nil { + t.Fatal(err) + } + compare(t, want, got) + }) + } +} + +func TestImpersonated(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + want, err := current() + if err != nil { + t.Fatal(err) + } + + // Create a test user and log in as that user. + token, _ := windowsTestAccount(t) + + // Impersonate the test user. + if err = windows.ImpersonateLoggedOnUser(token); err != nil { + t.Fatal(err) + } + defer func() { + err = windows.RevertToSelf() + if err != nil { + // If we can't revert to self, we can't continue testing. + panic(err) + } + }() + + got, err := current() + if err != nil { + t.Fatal(err) + } + compare(t, want, got) +} + +func TestCurrentNetapi32(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { + // Test that Current does not load netapi32.dll. + // First call Current. + Current() + + // Then check if netapi32.dll is loaded. + netapi32, err := syscall.UTF16PtrFromString("netapi32.dll") + if err != nil { + fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) + os.Exit(9) + return + } + mod, _ := windows.GetModuleHandle(netapi32) + if mod != 0 { + fmt.Fprintf(os.Stderr, "netapi32.dll is loaded\n") + os.Exit(9) + return + } + os.Exit(0) + return + } + exe := testenv.Executable(t) + cmd := testenv.CleanCmdEnv(exec.Command(exe, "-test.run=^TestCurrentNetapi32$")) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } +} + +func TestGroupIdsTestUser(t *testing.T) { + // Create a test user and log in as that user. + _, user := windowsTestAccount(t) + + gids, err := user.GroupIds() + if err != nil { + t.Fatal(err) + } + + if err != nil { + t.Fatalf("%+v.GroupIds(): %v", user, err) + } + if !containsID(gids, user.Gid) { + t.Errorf("%+v.GroupIds() = %v; does not contain user GID %s", user, gids, user.Gid) + } +} + +var serviceAccounts = []struct { + sid string + name string +}{ + {"S-1-5-18", "NT AUTHORITY\\SYSTEM"}, + {"S-1-5-19", "NT AUTHORITY\\LOCAL SERVICE"}, + {"S-1-5-20", "NT AUTHORITY\\NETWORK SERVICE"}, +} + +func TestLookupServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + u, err := Lookup(tt.name) + if err != nil { + t.Errorf("Lookup(%q): %v", tt.name, err) + continue + } + if u.Uid != tt.sid { + t.Errorf("unexpected uid for %q; got %q, want %q", u.Name, u.Uid, tt.sid) + } + } +} + +func TestLookupIdServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + u, err := LookupId(tt.sid) + if err != nil { + t.Errorf("LookupId(%q): %v", tt.sid, err) + continue + } + if u.Gid != tt.sid { + t.Errorf("unexpected gid for %q; got %q, want %q", u.Name, u.Gid, tt.sid) + } + if u.Username != tt.name { + t.Errorf("unexpected user name for %q; got %q, want %q", u.Gid, u.Username, tt.name) + } + } +} + +func TestLookupGroupServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + u, err := LookupGroup(tt.name) + if err != nil { + t.Errorf("LookupGroup(%q): %v", tt.name, err) + continue + } + if u.Gid != tt.sid { + t.Errorf("unexpected gid for %q; got %q, want %q", u.Name, u.Gid, tt.sid) + } + } +} + +func TestLookupGroupIdServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + u, err := LookupGroupId(tt.sid) + if err != nil { + t.Errorf("LookupGroupId(%q): %v", tt.sid, err) + continue + } + if u.Gid != tt.sid { + t.Errorf("unexpected gid for %q; got %q, want %q", u.Name, u.Gid, tt.sid) + } + } +} diff --git a/src/os/wait_wait6.go b/src/os/wait_wait6.go index 10314288..113535e5 100644 --- a/src/os/wait_wait6.go +++ b/src/os/wait_wait6.go @@ -15,18 +15,18 @@ import ( // succeed immediately, and reports whether it has done so. // It does not actually call p.Wait. func (p *Process) blockUntilWaitable() (bool, error) { - var errno syscall.Errno - for { - _, errno = wait6(_P_PID, p.Pid, syscall.WEXITED|syscall.WNOWAIT) - if errno != syscall.EINTR { - break + err := ignoringEINTR(func() error { + _, errno := wait6(_P_PID, p.Pid, syscall.WEXITED|syscall.WNOWAIT) + if errno != 0 { + return errno } - } + return nil + }) runtime.KeepAlive(p) - if errno == syscall.ENOSYS { + if err == syscall.ENOSYS { return false, nil - } else if errno != 0 { - return false, NewSyscallError("wait6", errno) + } else if err != nil { + return false, NewSyscallError("wait6", err) } return true, nil } diff --git a/src/os/wait_waitid.go b/src/os/wait_waitid.go index cd078f35..832dbe90 100644 --- a/src/os/wait_waitid.go +++ b/src/os/wait_waitid.go @@ -10,39 +10,28 @@ package os import ( + "internal/syscall/unix" "runtime" "syscall" - "unsafe" ) -const _P_PID = 1 - // blockUntilWaitable attempts to block until a call to p.Wait will // succeed immediately, and reports whether it has done so. // It does not actually call p.Wait. func (p *Process) blockUntilWaitable() (bool, error) { - // The waitid system call expects a pointer to a siginfo_t, - // which is 128 bytes on all Linux systems. - // On darwin/amd64, it requires 104 bytes. - // We don't care about the values it returns. - var siginfo [16]uint64 - psig := &siginfo[0] - var e syscall.Errno - for { - _, _, e = syscall.Syscall6(syscall.SYS_WAITID, _P_PID, uintptr(p.Pid), uintptr(unsafe.Pointer(psig)), syscall.WEXITED|syscall.WNOWAIT, 0, 0) - if e != syscall.EINTR { - break - } - } + var info unix.SiginfoChild + err := ignoringEINTR(func() error { + return unix.Waitid(unix.P_PID, p.Pid, &info, syscall.WEXITED|syscall.WNOWAIT, nil) + }) runtime.KeepAlive(p) - if e != 0 { + if err != nil { // waitid has been available since Linux 2.6.9, but // reportedly is not available in Ubuntu on Windows. // See issue 16610. - if e == syscall.ENOSYS { + if err == syscall.ENOSYS { return false, nil } - return false, NewSyscallError("waitid", e) + return false, NewSyscallError("waitid", err) } return true, nil } diff --git a/src/os/writeto_linux_test.go b/src/os/writeto_linux_test.go index e3900631..59caecd0 100644 --- a/src/os/writeto_linux_test.go +++ b/src/os/writeto_linux_test.go @@ -8,13 +8,11 @@ import ( "bytes" "internal/poll" "io" - "math/rand" "net" . "os" "strconv" "syscall" "testing" - "time" ) func TestSendFile(t *testing.T) { @@ -102,7 +100,7 @@ func newSendFileTest(t *testing.T, proto string, size int64) (net.Conn, *File, n hook := hookSendFile(t) client, server := createSocketPair(t, proto) - tempFile, data := createTempFile(t, size) + tempFile, data := createTempFile(t, "writeto-sendfile-to-socket", size) return client, tempFile, server, data, hook } @@ -133,30 +131,3 @@ type sendFileHook struct { handled bool err error } - -func createTempFile(t *testing.T, size int64) (*File, []byte) { - f, err := CreateTemp(t.TempDir(), "writeto-sendfile-to-socket") - if err != nil { - t.Fatalf("failed to create temporary file: %v", err) - } - t.Cleanup(func() { - f.Close() - }) - - randSeed := time.Now().Unix() - t.Logf("random data seed: %d\n", randSeed) - prng := rand.New(rand.NewSource(randSeed)) - data := make([]byte, size) - prng.Read(data) - if _, err := f.Write(data); err != nil { - t.Fatalf("failed to create and feed the file: %v", err) - } - if err := f.Sync(); err != nil { - t.Fatalf("failed to save the file: %v", err) - } - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to rewind the file: %v", err) - } - - return f, data -} diff --git a/src/os/zero_copy_freebsd.go b/src/os/zero_copy_freebsd.go new file mode 100644 index 00000000..aacfe861 --- /dev/null +++ b/src/os/zero_copy_freebsd.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "internal/poll" + "io" +) + +var pollCopyFileRange = poll.CopyFileRange + +func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { + return 0, false, nil +} + +func (f *File) readFrom(r io.Reader) (written int64, handled bool, err error) { + // copy_file_range(2) doesn't support destinations opened with + // O_APPEND, so don't bother to try zero-copy with these system calls. + // + // Visit https://man.freebsd.org/cgi/man.cgi?copy_file_range(2)#ERRORS for details. + if f.appendMode { + return 0, false, nil + } + + var ( + remain int64 + lr *io.LimitedReader + ) + if lr, r, remain = tryLimitedReader(r); remain <= 0 { + return 0, true, nil + } + + var src *File + switch v := r.(type) { + case *File: + src = v + case fileWithoutWriteTo: + src = v.File + default: + return 0, false, nil + } + + if src.checkValid("ReadFrom") != nil { + // Avoid returning the error as we report handled as false, + // leave further error handling as the responsibility of the caller. + return 0, false, nil + } + + written, handled, err = pollCopyFileRange(&f.pfd, &src.pfd, remain) + if lr != nil { + lr.N -= written + } + + return written, handled, wrapSyscallError("copy_file_range", err) +} diff --git a/src/os/zero_copy_linux.go b/src/os/zero_copy_linux.go index 0afc19e1..9d666a3c 100644 --- a/src/os/zero_copy_linux.go +++ b/src/os/zero_copy_linux.go @@ -15,15 +15,6 @@ var ( pollSplice = poll.Splice ) -// wrapSyscallError takes an error and a syscall name. If the error is -// a syscall.Errno, it wraps it in an os.SyscallError using the syscall name. -func wrapSyscallError(name string, err error) error { - if _, ok := err.(syscall.Errno); ok { - err = NewSyscallError(name, err) - } - return err -} - func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { pfd, network := getPollFDAndNetwork(w) // TODO(panjf2000): same as File.spliceToFile. @@ -37,7 +28,7 @@ func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { } rerr := sc.Read(func(fd uintptr) (done bool) { - written, err, handled = poll.SendFile(pfd, int(fd), 1<<63-1) + written, err, handled = poll.SendFile(pfd, int(fd), 0) return true }) @@ -149,21 +140,6 @@ func getPollFDAndNetwork(i any) (*poll.FD, poll.String) { return irc.PollFD(), irc.Network() } -// tryLimitedReader tries to assert the io.Reader to io.LimitedReader, it returns the io.LimitedReader, -// the underlying io.Reader and the remaining amount of bytes if the assertion succeeds, -// otherwise it just returns the original io.Reader and the theoretical unlimited remaining amount of bytes. -func tryLimitedReader(r io.Reader) (*io.LimitedReader, io.Reader, int64) { - var remain int64 = 1<<63 - 1 // by default, copy until EOF - - lr, ok := r.(*io.LimitedReader) - if !ok { - return nil, r, remain - } - - remain = lr.N - return lr, lr.R, remain -} - func isUnixOrTCP(network string) bool { switch network { case "tcp", "tcp4", "tcp6", "unix": diff --git a/src/os/zero_copy_posix.go b/src/os/zero_copy_posix.go new file mode 100644 index 00000000..161144fe --- /dev/null +++ b/src/os/zero_copy_posix.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || js || wasip1 || windows + +package os + +import ( + "io" + "syscall" +) + +// wrapSyscallError takes an error and a syscall name. If the error is +// a syscall.Errno, it wraps it in an os.SyscallError using the syscall name. +func wrapSyscallError(name string, err error) error { + if _, ok := err.(syscall.Errno); ok { + err = NewSyscallError(name, err) + } + return err +} + +// tryLimitedReader tries to assert the io.Reader to io.LimitedReader, it returns the io.LimitedReader, +// the underlying io.Reader and the remaining amount of bytes if the assertion succeeds, +// otherwise it just returns the original io.Reader and the theoretical unlimited remaining amount of bytes. +func tryLimitedReader(r io.Reader) (*io.LimitedReader, io.Reader, int64) { + var remain int64 = 1<<63 - 1 // by default, copy until EOF + + lr, ok := r.(*io.LimitedReader) + if !ok { + return nil, r, remain + } + + remain = lr.N + return lr, lr.R, remain +} diff --git a/src/os/zero_copy_solaris.go b/src/os/zero_copy_solaris.go new file mode 100644 index 00000000..94a8de60 --- /dev/null +++ b/src/os/zero_copy_solaris.go @@ -0,0 +1,92 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "internal/poll" + "io" + "runtime" + "syscall" +) + +func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { + return 0, false, nil +} + +// readFrom is basically a refactor of net.sendFile, but adapted to work for the target of *File. +func (f *File) readFrom(r io.Reader) (written int64, handled bool, err error) { + var remain int64 = 0 // 0 indicates sending until EOF + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, true, nil + } + } + + var src *File + switch v := r.(type) { + case *File: + src = v + case fileWithoutWriteTo: + src = v.File + default: + return 0, false, nil + } + + if src.checkValid("ReadFrom") != nil { + // Avoid returning the error as we report handled as false, + // leave further error handling as the responsibility of the caller. + return 0, false, nil + } + + // If fd_in and fd_out refer to the same file and the source and target ranges overlap, + // sendfile(2) on SunOS will allow this kind of overlapping and work like a memmove, + // in this case the file content remains the same after copying, which is not what we want. + // Thus, we just bail out here and leave it to generic copy when it's a file copying itself. + if f.pfd.Sysfd == src.pfd.Sysfd { + return 0, false, nil + } + + // sendfile() on illumos seems to incur intermittent failures when the + // target file is a standard stream (stdout/stderr), we hereby skip any + // anything other than regular files conservatively and leave them to generic copy. + // Check out https://go.dev/issue/68863 for more details. + if runtime.GOOS == "illumos" { + fi, err := f.Stat() + if err != nil { + return 0, false, nil + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false, nil + } + if typ := st.Mode & syscall.S_IFMT; typ != syscall.S_IFREG { + return 0, false, nil + } + } + + sc, err := src.SyscallConn() + if err != nil { + return + } + + // System call sendfile()s on Solaris and illumos support file-to-file copying. + // Check out https://docs.oracle.com/cd/E86824_01/html/E54768/sendfile-3ext.html and + // https://docs.oracle.com/cd/E88353_01/html/E37843/sendfile-3c.html and + // https://illumos.org/man/3EXT/sendfile for more details. + rerr := sc.Read(func(fd uintptr) bool { + written, err, handled = poll.SendFile(&f.pfd, int(fd), remain) + return true + }) + if lr != nil { + lr.N = remain - written + } + if err == nil { + err = rerr + } + + return written, handled, wrapSyscallError("sendfile", err) +} diff --git a/src/os/zero_copy_stub.go b/src/os/zero_copy_stub.go index 9ec58081..0470a205 100644 --- a/src/os/zero_copy_stub.go +++ b/src/os/zero_copy_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !linux +//go:build !freebsd && !linux && !solaris package os diff --git a/src/path/filepath/match_test.go b/src/path/filepath/match_test.go index c6b1c505..f415b040 100644 --- a/src/path/filepath/match_test.go +++ b/src/path/filepath/match_test.go @@ -9,7 +9,6 @@ import ( "internal/testenv" "os" . "path/filepath" - "reflect" "runtime" "slices" "strings" @@ -327,20 +326,7 @@ func TestWindowsGlob(t *testing.T) { } // test relative paths - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - err = os.Chdir(tmpDir) - if err != nil { - t.Fatal(err) - } - defer func() { - err := os.Chdir(wd) - if err != nil { - t.Fatal(err) - } - }() + t.Chdir(tmpDir) for _, test := range tests { err := test.globRel("") if err != nil { @@ -367,7 +353,7 @@ func TestNonWindowsGlobEscape(t *testing.T) { if err != nil { t.Fatalf("Glob error for %q: %s", pattern, err) } - if !reflect.DeepEqual(matches, want) { + if !slices.Equal(matches, want) { t.Fatalf("Glob(%#q) = %v want %v", pattern, matches, want) } } diff --git a/src/path/filepath/path_test.go b/src/path/filepath/path_test.go index b24f39c5..e9cd82d6 100644 --- a/src/path/filepath/path_test.go +++ b/src/path/filepath/path_test.go @@ -364,7 +364,7 @@ func TestSplitList(t *testing.T) { tests = append(tests, winsplitlisttests...) } for _, test := range tests { - if l := filepath.SplitList(test.list); !reflect.DeepEqual(l, test.result) { + if l := filepath.SplitList(test.list); !slices.Equal(l, test.result) { t.Errorf("SplitList(%#q) = %#q, want %#q", test.list, l, test.result) } } @@ -597,45 +597,6 @@ func mark(d fs.DirEntry, err error, errors *[]error, clear bool) error { return nil } -// chdir changes the current working directory to the named directory, -// and then restore the original working directory at the end of the test. -func chdir(t *testing.T, dir string) { - olddir, err := os.Getwd() - if err != nil { - t.Fatalf("getwd %s: %v", dir, err) - } - if err := os.Chdir(dir); err != nil { - t.Fatalf("chdir %s: %v", dir, err) - } - - t.Cleanup(func() { - if err := os.Chdir(olddir); err != nil { - t.Errorf("restore original working directory %s: %v", olddir, err) - os.Exit(1) - } - }) -} - -func chtmpdir(t *testing.T) (restore func()) { - oldwd, err := os.Getwd() - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - d, err := os.MkdirTemp("", "test") - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - if err := os.Chdir(d); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - return func() { - if err := os.Chdir(oldwd); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - os.RemoveAll(d) - } -} - // tempDirCanonical returns a temporary directory for the test to use, ensuring // that the returned path does not contain symlinks. func tempDirCanonical(t *testing.T) string { @@ -663,21 +624,7 @@ func TestWalkDir(t *testing.T) { } func testWalk(t *testing.T, walk func(string, fs.WalkDirFunc) error, errVisit int) { - if runtime.GOOS == "ios" { - restore := chtmpdir(t) - defer restore() - } - - tmpDir := t.TempDir() - - origDir, err := os.Getwd() - if err != nil { - t.Fatal("finding working dir:", err) - } - if err = os.Chdir(tmpDir); err != nil { - t.Fatal("entering temp dir:", err) - } - defer os.Chdir(origDir) + t.Chdir(t.TempDir()) makeTree(t) errors := make([]error, 0, 10) @@ -686,7 +633,7 @@ func testWalk(t *testing.T, walk func(string, fs.WalkDirFunc) error, errVisit in return mark(d, err, &errors, clear) } // Expect no errors. - err = walk(tree.name, markFn) + err := walk(tree.name, markFn) if err != nil { t.Fatalf("no error expected, found: %s", err) } @@ -1004,7 +951,7 @@ func TestWalkSymlinkRoot(t *testing.T) { t.Fatal(err) } - if !reflect.DeepEqual(walked, tt.want) { + if !slices.Equal(walked, tt.want) { t.Logf("Walk(%#q) visited %#q; want %#q", tt.root, walked, tt.want) if slices.Contains(tt.buggyGOOS, runtime.GOOS) { t.Logf("(ignoring known bug on %v)", runtime.GOOS) @@ -1225,22 +1172,7 @@ func testEvalSymlinks(t *testing.T, path, want string) { } func testEvalSymlinksAfterChdir(t *testing.T, wd, path, want string) { - cwd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - defer func() { - err := os.Chdir(cwd) - if err != nil { - t.Fatal(err) - } - }() - - err = os.Chdir(wd) - if err != nil { - t.Fatal(err) - } - + t.Chdir(wd) have, err := filepath.EvalSymlinks(path) if err != nil { t.Errorf("EvalSymlinks(%q) in %q directory error: %v", path, wd, err) @@ -1314,8 +1246,7 @@ func TestEvalSymlinks(t *testing.T) { func TestEvalSymlinksIsNotExist(t *testing.T) { testenv.MustHaveSymlink(t) - - defer chtmpdir(t)() + t.Chdir(t.TempDir()) _, err := filepath.EvalSymlinks("notexist") if !os.IsNotExist(err) { @@ -1396,10 +1327,10 @@ func TestIssue13582(t *testing.T) { // Issue 57905. func TestRelativeSymlinkToAbsolute(t *testing.T) { testenv.MustHaveSymlink(t) - // Not parallel: uses os.Chdir. + // Not parallel: uses t.Chdir. tmpDir := t.TempDir() - chdir(t, tmpDir) + t.Chdir(tmpDir) // Create "link" in the current working directory as a symlink to an arbitrary // absolute path. On macOS, this path is likely to begin with a symlink @@ -1452,18 +1383,10 @@ var absTests = []string{ func TestAbs(t *testing.T) { root := t.TempDir() - wd, err := os.Getwd() - if err != nil { - t.Fatal("getwd failed: ", err) - } - err = os.Chdir(root) - if err != nil { - t.Fatal("chdir failed: ", err) - } - defer os.Chdir(wd) + t.Chdir(root) for _, dir := range absTestDirs { - err = os.Mkdir(dir, 0777) + err := os.Mkdir(dir, 0777) if err != nil { t.Fatal("Mkdir failed: ", err) } @@ -1485,7 +1408,7 @@ func TestAbs(t *testing.T) { tests = append(slices.Clip(tests), extra...) } - err = os.Chdir(absTestDirs[0]) + err := os.Chdir(absTestDirs[0]) if err != nil { t.Fatal("chdir failed: ", err) } @@ -1521,16 +1444,7 @@ func TestAbs(t *testing.T) { // a valid path, so it can't be used with os.Stat. func TestAbsEmptyString(t *testing.T) { root := t.TempDir() - - wd, err := os.Getwd() - if err != nil { - t.Fatal("getwd failed: ", err) - } - err = os.Chdir(root) - if err != nil { - t.Fatal("chdir failed: ", err) - } - defer os.Chdir(wd) + t.Chdir(root) info, err := os.Stat(root) if err != nil { @@ -1757,19 +1671,9 @@ func TestBug3486(t *testing.T) { // https://golang.org/issue/3486 func testWalkSymlink(t *testing.T, mklink func(target, link string) error) { tmpdir := t.TempDir() + t.Chdir(tmpdir) - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - defer os.Chdir(wd) - - err = os.Chdir(tmpdir) - if err != nil { - t.Fatal(err) - } - - err = mklink(tmpdir, "link") + err := mklink(tmpdir, "link") if err != nil { t.Fatal(err) } @@ -1882,13 +1786,7 @@ func TestEvalSymlinksAboveRoot(t *testing.T) { // Issue 30520 part 2. func TestEvalSymlinksAboveRootChdir(t *testing.T) { testenv.MustHaveSymlink(t) - - tmpDir, err := os.MkdirTemp("", "TestEvalSymlinksAboveRootChdir") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - chdir(t, tmpDir) + t.Chdir(t.TempDir()) subdir := filepath.Join("a", "b") if err := os.MkdirAll(subdir, 0777); err != nil { @@ -1950,18 +1848,17 @@ func TestIssue51617(t *testing.T) { t.Fatal(err) } want := []string{".", "a", filepath.Join("a", "bad"), filepath.Join("a", "next")} - if !reflect.DeepEqual(saw, want) { + if !slices.Equal(saw, want) { t.Errorf("got directories %v, want %v", saw, want) } } func TestEscaping(t *testing.T) { - dir1 := t.TempDir() - dir2 := t.TempDir() - chdir(t, dir1) + dir := t.TempDir() + t.Chdir(t.TempDir()) for _, p := range []string{ - filepath.Join(dir2, "x"), + filepath.Join(dir, "x"), } { if !filepath.IsLocal(p) { continue @@ -1970,7 +1867,7 @@ func TestEscaping(t *testing.T) { if err != nil { f.Close() } - ents, err := os.ReadDir(dir2) + ents, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } diff --git a/src/path/filepath/path_windows.go b/src/path/filepath/path_windows.go index d53f87f1..d0eb42c5 100644 --- a/src/path/filepath/path_windows.go +++ b/src/path/filepath/path_windows.go @@ -1,3 +1,7 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package filepath import ( diff --git a/src/path/filepath/path_windows_test.go b/src/path/filepath/path_windows_test.go index 2862f390..d60903f6 100644 --- a/src/path/filepath/path_windows_test.go +++ b/src/path/filepath/path_windows_test.go @@ -13,8 +13,8 @@ import ( "os" "os/exec" "path/filepath" - "reflect" "runtime/debug" + "slices" "strings" "testing" ) @@ -83,7 +83,7 @@ func testWinSplitListTestIsValid(t *testing.T, ti int, tt SplitListTest, case err != nil: t.Errorf("%d,%d: execution error %v\n%q", ti, i, err, out) return - case !reflect.DeepEqual(out, exp): + case !slices.Equal(out, exp): t.Errorf("%d,%d: expected %#q, got %#q", ti, i, exp, out) return default: @@ -408,12 +408,7 @@ func TestToNorm(t *testing.T) { if err != nil { t.Fatal(err) } - defer func() { - err := os.Chdir(cwd) - if err != nil { - t.Fatal(err) - } - }() + t.Chdir(".") // Ensure cwd is restored after the test. tmpVol := filepath.VolumeName(ctmp) if len(tmpVol) != 2 { diff --git a/src/plugin/plugin.go b/src/plugin/plugin.go index b4b1697b..a4ff694e 100644 --- a/src/plugin/plugin.go +++ b/src/plugin/plugin.go @@ -29,6 +29,10 @@ // macOS, making them unsuitable for applications intended to be // portable. // +// - Plugins are poorly supported by the Go race detector. Even simple +// race conditions may not be automatically detected. See +// https://go.dev/issue/24245 for more information. +// // - Applications that use plugins may require careful configuration // to ensure that the various parts of the program be made available // in the correct location in the file system (or container image). diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index a57f17f8..b2f70c13 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -10,8 +10,9 @@ import ( "flag" "fmt" "go/token" - "internal/abi" + "internal/asan" "internal/goarch" + "internal/goexperiment" "internal/testenv" "io" "math" @@ -22,6 +23,7 @@ import ( "reflect/internal/example1" "reflect/internal/example2" "runtime" + "runtime/debug" "slices" "strconv" "strings" @@ -32,8 +34,6 @@ import ( "unsafe" ) -const bucketCount = abi.MapBucketCount - var sink any func TestBool(t *testing.T) { @@ -1134,13 +1134,15 @@ var deepEqualTests = []DeepEqualTest{ } func TestDeepEqual(t *testing.T) { - for _, test := range deepEqualTests { - if test.b == (self{}) { - test.b = test.a - } - if r := DeepEqual(test.a, test.b); r != test.eq { - t.Errorf("DeepEqual(%#v, %#v) = %v, want %v", test.a, test.b, r, test.eq) - } + for i, test := range deepEqualTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + if test.b == (self{}) { + test.b = test.a + } + if r := DeepEqual(test.a, test.b); r != test.eq { + t.Errorf("DeepEqual(%#v, %#v) = %v, want %v", test.a, test.b, r, test.eq) + } + }) } } @@ -1273,6 +1275,14 @@ var deepEqualPerfTests = []struct { } func TestDeepEqualAllocs(t *testing.T) { + // TODO(prattmic): maps on stack + if goexperiment.SwissMap { + t.Skipf("Maps on stack not yet implemented") + } + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } + for _, tt := range deepEqualPerfTests { t.Run(ValueOf(tt.x).Type().String(), func(t *testing.T) { got := testing.AllocsPerRun(100, func() { @@ -3533,6 +3543,14 @@ func TestAllocations(t *testing.T) { panic("wrong result") } }) + if runtime.GOOS != "js" && runtime.GOOS != "wasip1" { + typ := TypeFor[struct{ f int }]() + noAlloc(t, 100, func(int) { + if typ.Field(0).Index[0] != 0 { + panic("wrong field index") + } + }) + } } func TestSmallNegativeInt(t *testing.T) { @@ -6291,6 +6309,32 @@ func TestMapOfGCKeys(t *testing.T) { } } +// Test assignment and access to a map with keys larger than word size. +func TestMapOfGCBigKey(t *testing.T) { + type KV struct { + i int64 + j int64 + } + + kvTyp := TypeFor[KV]() + mt := MapOf(kvTyp, kvTyp) + + const n = 100 + m := MakeMap(mt) + for i := 0; i < n; i++ { + kv := KV{int64(i), int64(i+1)} + m.SetMapIndex(ValueOf(kv), ValueOf(kv)) + } + + for i := 0; i < n; i++ { + kv := KV{int64(i), int64(i+1)} + elem := m.MapIndex(ValueOf(kv)).Interface().(KV) + if elem != kv { + t.Errorf("lost m[%v] = %v, want %v", kv, elem, kv) + } + } +} + func TestMapOfGCValues(t *testing.T) { type T *uintptr tt := TypeOf(T(nil)) @@ -6823,7 +6867,7 @@ func TestInvalid(t *testing.T) { } // Issue 8917. -func TestLargeGCProg(t *testing.T) { +func TestLarge(t *testing.T) { fv := ValueOf(func([256]*byte) {}) fv.Call([]Value{ValueOf([256]*byte{})}) } @@ -6863,6 +6907,25 @@ func TestTypeFieldOutOfRangePanic(t *testing.T) { } } +func TestTypeFieldReadOnly(t *testing.T) { + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + // This is OK because we don't use the optimization + // for js or wasip1. + t.Skip("test does not fault on GOOS=js") + } + + // It's important that changing one StructField.Index + // value not affect other StructField.Index values. + // Right now StructField.Index is read-only; + // that saves allocations but is otherwise not important. + typ := TypeFor[struct{ f int }]() + f := typ.Field(0) + defer debug.SetPanicOnFault(debug.SetPanicOnFault(true)) + shouldPanic("", func() { + f.Index[0] = 1 + }) +} + // Issue 9179. func TestCallGC(t *testing.T) { f := func(a, b, c, d, e string) { @@ -7144,60 +7207,61 @@ func verifyGCBitsSlice(t *testing.T, typ Type, cap int, bits []byte) { t.Errorf("line %d: heapBits incorrect for make(%v, 0, %v)\nhave %v\nwant %v", line, typ, cap, heapBits, bits) } -func TestGCBits(t *testing.T) { - verifyGCBits(t, TypeOf((*byte)(nil)), []byte{1}) +// Building blocks for types seen by the compiler (like [2]Xscalar). +// The compiler will create the type structures for the derived types, +// including their GC metadata. +type Xscalar struct{ x uintptr } +type Xptr struct{ x *byte } +type Xptrscalar struct { + *byte + uintptr +} +type Xscalarptr struct { + uintptr + *byte +} +type Xbigptrscalar struct { + _ [100]*byte + _ [100]uintptr +} - // Building blocks for types seen by the compiler (like [2]Xscalar). - // The compiler will create the type structures for the derived types, - // including their GC metadata. - type Xscalar struct{ x uintptr } - type Xptr struct{ x *byte } - type Xptrscalar struct { +var Tscalar, Tint64, Tptr, Tscalarptr, Tptrscalar, Tbigptrscalar Type + +func init() { + // Building blocks for types constructed by reflect. + // This code is in a separate block so that code below + // cannot accidentally refer to these. + // The compiler must NOT see types derived from these + // (for example, [2]Scalar must NOT appear in the program), + // or else reflect will use it instead of having to construct one. + // The goal is to test the construction. + type Scalar struct{ x uintptr } + type Ptr struct{ x *byte } + type Ptrscalar struct { *byte uintptr } - type Xscalarptr struct { + type Scalarptr struct { uintptr *byte } - type Xbigptrscalar struct { + type Bigptrscalar struct { _ [100]*byte _ [100]uintptr } + type Int64 int64 + Tscalar = TypeOf(Scalar{}) + Tint64 = TypeOf(Int64(0)) + Tptr = TypeOf(Ptr{}) + Tscalarptr = TypeOf(Scalarptr{}) + Tptrscalar = TypeOf(Ptrscalar{}) + Tbigptrscalar = TypeOf(Bigptrscalar{}) +} - var Tscalar, Tint64, Tptr, Tscalarptr, Tptrscalar, Tbigptrscalar Type - { - // Building blocks for types constructed by reflect. - // This code is in a separate block so that code below - // cannot accidentally refer to these. - // The compiler must NOT see types derived from these - // (for example, [2]Scalar must NOT appear in the program), - // or else reflect will use it instead of having to construct one. - // The goal is to test the construction. - type Scalar struct{ x uintptr } - type Ptr struct{ x *byte } - type Ptrscalar struct { - *byte - uintptr - } - type Scalarptr struct { - uintptr - *byte - } - type Bigptrscalar struct { - _ [100]*byte - _ [100]uintptr - } - type Int64 int64 - Tscalar = TypeOf(Scalar{}) - Tint64 = TypeOf(Int64(0)) - Tptr = TypeOf(Ptr{}) - Tscalarptr = TypeOf(Scalarptr{}) - Tptrscalar = TypeOf(Ptrscalar{}) - Tbigptrscalar = TypeOf(Bigptrscalar{}) - } +var empty = []byte{} - empty := []byte{} +func TestGCBits(t *testing.T) { + verifyGCBits(t, TypeOf((*byte)(nil)), []byte{1}) verifyGCBits(t, TypeOf(Xscalar{}), empty) verifyGCBits(t, Tscalar, empty) @@ -7277,47 +7341,7 @@ func TestGCBits(t *testing.T) { verifyGCBits(t, TypeOf(([][10000]Xscalar)(nil)), lit(1)) verifyGCBits(t, SliceOf(ArrayOf(10000, Tscalar)), lit(1)) - hdr := make([]byte, bucketCount/goarch.PtrSize) - - verifyMapBucket := func(t *testing.T, k, e Type, m any, want []byte) { - verifyGCBits(t, MapBucketOf(k, e), want) - verifyGCBits(t, CachedBucketOf(TypeOf(m)), want) - } - verifyMapBucket(t, - Tscalar, Tptr, - map[Xscalar]Xptr(nil), - join(hdr, rep(bucketCount, lit(0)), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, - Tscalarptr, Tptr, - map[Xscalarptr]Xptr(nil), - join(hdr, rep(bucketCount, lit(0, 1)), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, Tint64, Tptr, - map[int64]Xptr(nil), - join(hdr, rep(bucketCount, rep(8/goarch.PtrSize, lit(0))), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, - Tscalar, Tscalar, - map[Xscalar]Xscalar(nil), - empty) - verifyMapBucket(t, - ArrayOf(2, Tscalarptr), ArrayOf(3, Tptrscalar), - map[[2]Xscalarptr][3]Xptrscalar(nil), - join(hdr, rep(bucketCount*2, lit(0, 1)), rep(bucketCount*3, lit(1, 0)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize, Tscalarptr), ArrayOf(64/goarch.PtrSize, Tptrscalar), - map[[64 / goarch.PtrSize]Xscalarptr][64 / goarch.PtrSize]Xptrscalar(nil), - join(hdr, rep(bucketCount*64/goarch.PtrSize, lit(0, 1)), rep(bucketCount*64/goarch.PtrSize, lit(1, 0)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize+1, Tscalarptr), ArrayOf(64/goarch.PtrSize, Tptrscalar), - map[[64/goarch.PtrSize + 1]Xscalarptr][64 / goarch.PtrSize]Xptrscalar(nil), - join(hdr, rep(bucketCount, lit(1)), rep(bucketCount*64/goarch.PtrSize, lit(1, 0)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize, Tscalarptr), ArrayOf(64/goarch.PtrSize+1, Tptrscalar), - map[[64 / goarch.PtrSize]Xscalarptr][64/goarch.PtrSize + 1]Xptrscalar(nil), - join(hdr, rep(bucketCount*64/goarch.PtrSize, lit(0, 1)), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize+1, Tscalarptr), ArrayOf(64/goarch.PtrSize+1, Tptrscalar), - map[[64/goarch.PtrSize + 1]Xscalarptr][64/goarch.PtrSize + 1]Xptrscalar(nil), - join(hdr, rep(bucketCount, lit(1)), rep(bucketCount, lit(1)), lit(1))) + testGCBitsMap(t) } func rep(n int, b []byte) []byte { return bytes.Repeat(b, n) } @@ -7359,6 +7383,9 @@ func TestPtrToMethods(t *testing.T) { } func TestMapAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } m := ValueOf(make(map[int]int, 10)) k := ValueOf(5) v := ValueOf(7) @@ -7389,6 +7416,9 @@ func TestMapAlloc(t *testing.T) { } func TestChanAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } // Note: for a chan int, the return Value must be allocated, so we // use a chan *int instead. c := ValueOf(make(chan *int, 1)) @@ -7751,11 +7781,14 @@ func TestMapIterReset(t *testing.T) { } // Reset should not allocate. + // + // Except with -asan, where there are additional allocations. + // See #70079. n := int(testing.AllocsPerRun(10, func() { iter.Reset(ValueOf(m2)) iter.Reset(Value{}) })) - if n > 0 { + if !asan.Enabled && n > 0 { t.Errorf("MapIter.Reset allocated %d times", n) } } @@ -8603,3 +8636,48 @@ func TestSliceAt(t *testing.T) { // _ = SliceAt(typ, unsafe.Pointer(last), 1) shouldPanic("", func() { _ = SliceAt(typ, unsafe.Pointer(last), 2) }) } + +// Test that maps created with MapOf properly updates keys on overwrite as +// expected (i.e., it sets the key update flag in the map). +// +// This test is based on runtime.TestNegativeZero. +func TestMapOfKeyUpdate(t *testing.T) { + m := MakeMap(MapOf(TypeFor[float64](), TypeFor[bool]())) + + zero := float64(0.0) + negZero := math.Copysign(zero, -1.0) + + m.SetMapIndex(ValueOf(zero), ValueOf(true)) + m.SetMapIndex(ValueOf(negZero), ValueOf(true)) + + if m.Len() != 1 { + t.Errorf("map length got %d want 1", m.Len()) + } + + iter := m.MapRange() + for iter.Next() { + k := iter.Key().Float() + if math.Copysign(1.0, k) > 0 { + t.Errorf("map key %f has positive sign", k) + } + } +} + +// Test that maps created with MapOf properly panic on unhashable keys, even if +// the map is empty. (i.e., it sets the hash might panic flag in the map). +// +// This test is a simplified version of runtime.TestEmptyMapWithInterfaceKey +// for reflect. +func TestMapOfKeyPanic(t *testing.T) { + defer func() { + r := recover() + if r == nil { + t.Errorf("didn't panic") + } + }() + + m := MakeMap(MapOf(TypeFor[any](), TypeFor[bool]())) + + var slice []int + m.MapIndex(ValueOf(slice)) +} diff --git a/src/reflect/deepequal.go b/src/reflect/deepequal.go index 502ea9f1..041c3e1f 100644 --- a/src/reflect/deepequal.go +++ b/src/reflect/deepequal.go @@ -41,7 +41,7 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool) bool { case Pointer: if !v1.typ().Pointers() { // not-in-heap pointers can't be cyclic. - // At least, all of our current uses of runtime/internal/sys.NotInHeap + // At least, all of our current uses of internal/runtime/sys.NotInHeap // have that property. The runtime ones aren't cyclic (and we don't use // DeepEqual on them anyway), and the cgo-generated ones are // all empty structs. diff --git a/src/reflect/export_noswiss_test.go b/src/reflect/export_noswiss_test.go new file mode 100644 index 00000000..34e5e920 --- /dev/null +++ b/src/reflect/export_noswiss_test.go @@ -0,0 +1,25 @@ +// Copyright 2024 Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +package reflect + +import ( + "internal/abi" + "unsafe" +) + +func MapBucketOf(x, y Type) Type { + return toType(bucketOf(x.common(), y.common())) +} + +func CachedBucketOf(m Type) Type { + t := m.(*rtype) + if Kind(t.t.Kind_&abi.KindMask) != Map { + panic("not map") + } + tt := (*mapType)(unsafe.Pointer(t)) + return toType(tt.Bucket) +} diff --git a/src/reflect/export_swiss_test.go b/src/reflect/export_swiss_test.go new file mode 100644 index 00000000..ac3cd0ad --- /dev/null +++ b/src/reflect/export_swiss_test.go @@ -0,0 +1,12 @@ +// Copyright 2024 Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package reflect + +func MapGroupOf(x, y Type) Type { + grp, _ := groupAndSlotOf(x, y) + return grp +} diff --git a/src/reflect/export_test.go b/src/reflect/export_test.go index 30a0e823..eedd063f 100644 --- a/src/reflect/export_test.go +++ b/src/reflect/export_test.go @@ -58,9 +58,6 @@ func FuncLayout(t Type, rcvr Type) (frametype Type, argSize, retOffset uintptr, inReg = append(inReg, bool2byte(abid.inRegPtrs.Get(i))) outReg = append(outReg, bool2byte(abid.outRegPtrs.Get(i))) } - if ft.Kind_&abi.KindGCProg != 0 { - panic("can't handle gc programs") - } // Expand frame type's GC bitmap into byte-map. ptrs = ft.Pointers() @@ -91,19 +88,6 @@ var GCBits = gcbits func gcbits(any) []byte // provided by runtime -func MapBucketOf(x, y Type) Type { - return toType(bucketOf(x.common(), y.common())) -} - -func CachedBucketOf(m Type) Type { - t := m.(*rtype) - if Kind(t.t.Kind_&abi.KindMask) != Map { - panic("not map") - } - tt := (*mapType)(unsafe.Pointer(t)) - return toType(tt.Bucket) -} - type EmbedWithUnexpMeth struct{} func (EmbedWithUnexpMeth) f() {} diff --git a/src/reflect/iter_test.go b/src/reflect/iter_test.go index 9b78fcf7..00ae521a 100644 --- a/src/reflect/iter_test.go +++ b/src/reflect/iter_test.go @@ -114,10 +114,10 @@ func TestValueSeq(t *testing.T) { }}, {"string", ValueOf("12语言"), func(t *testing.T, s iter.Seq[Value]) { i := int64(0) - indexs := []int64{0, 1, 2, 5} + indexes := []int64{0, 1, 2, 5} for v := range s { - if v.Int() != indexs[i] { - t.Fatalf("got %d, want %d", v.Int(), indexs[i]) + if v.Int() != indexes[i] { + t.Fatalf("got %d, want %d", v.Int(), indexes[i]) } i++ } @@ -126,14 +126,12 @@ func TestValueSeq(t *testing.T) { } }}, {"map[string]int", ValueOf(m), func(t *testing.T, s iter.Seq[Value]) { - i := int64(0) copy := maps.Clone(m) for v := range s { if _, ok := copy[v.String()]; !ok { t.Fatalf("unexpected %v", v.Interface()) } delete(copy, v.String()) - i++ } if len(copy) != 0 { t.Fatalf("should loop four times") @@ -240,11 +238,10 @@ func TestValueSeq2(t *testing.T) { } }}, {"string", ValueOf("12语言"), func(t *testing.T, s iter.Seq2[Value, Value]) { - i := int64(0) - str := "12语言" next, stop := iter.Pull2(s) defer stop() - for j, s := range str { + i := int64(0) + for j, s := range "12语言" { v1, v2, ok := next() if !ok { t.Fatalf("should loop four times") diff --git a/src/reflect/map_noswiss.go b/src/reflect/map_noswiss.go new file mode 100644 index 00000000..19696a4f --- /dev/null +++ b/src/reflect/map_noswiss.go @@ -0,0 +1,484 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +package reflect + +import ( + "internal/abi" + "internal/goarch" + "unsafe" +) + +// mapType represents a map type. +type mapType struct { + abi.OldMapType +} + +// Pushed from runtime. + +//go:noescape +func mapiterinit(t *abi.Type, m unsafe.Pointer, it *hiter) + +//go:noescape +func mapiternext(it *hiter) + +func (t *rtype) Key() Type { + if t.Kind() != Map { + panic("reflect: Key of non-map type " + t.String()) + } + tt := (*mapType)(unsafe.Pointer(t)) + return toType(tt.Key) +} + +// MapOf returns the map type with the given key and element types. +// For example, if k represents int and e represents string, +// MapOf(k, e) represents map[int]string. +// +// If the key type is not a valid map key type (that is, if it does +// not implement Go's == operator), MapOf panics. +func MapOf(key, elem Type) Type { + ktyp := key.common() + etyp := elem.common() + + if ktyp.Equal == nil { + panic("reflect.MapOf: invalid key type " + stringFor(ktyp)) + } + + // Look in cache. + ckey := cacheKey{Map, ktyp, etyp, 0} + if mt, ok := lookupCache.Load(ckey); ok { + return mt.(Type) + } + + // Look in known types. + s := "map[" + stringFor(ktyp) + "]" + stringFor(etyp) + for _, tt := range typesByString(s) { + mt := (*mapType)(unsafe.Pointer(tt)) + if mt.Key == ktyp && mt.Elem == etyp { + ti, _ := lookupCache.LoadOrStore(ckey, toRType(tt)) + return ti.(Type) + } + } + + // Make a map type. + // Note: flag values must match those used in the TMAP case + // in ../cmd/compile/internal/reflectdata/reflect.go:writeType. + var imap any = (map[unsafe.Pointer]unsafe.Pointer)(nil) + mt := **(**mapType)(unsafe.Pointer(&imap)) + mt.Str = resolveReflectName(newName(s, "", false, false)) + mt.TFlag = 0 + mt.Hash = fnv1(etyp.Hash, 'm', byte(ktyp.Hash>>24), byte(ktyp.Hash>>16), byte(ktyp.Hash>>8), byte(ktyp.Hash)) + mt.Key = ktyp + mt.Elem = etyp + mt.Bucket = bucketOf(ktyp, etyp) + mt.Hasher = func(p unsafe.Pointer, seed uintptr) uintptr { + return typehash(ktyp, p, seed) + } + mt.Flags = 0 + if ktyp.Size_ > abi.OldMapMaxKeyBytes { + mt.KeySize = uint8(goarch.PtrSize) + mt.Flags |= 1 // indirect key + } else { + mt.KeySize = uint8(ktyp.Size_) + } + if etyp.Size_ > abi.OldMapMaxElemBytes { + mt.ValueSize = uint8(goarch.PtrSize) + mt.Flags |= 2 // indirect value + } else { + mt.ValueSize = uint8(etyp.Size_) + } + mt.BucketSize = uint16(mt.Bucket.Size_) + if isReflexive(ktyp) { + mt.Flags |= 4 + } + if needKeyUpdate(ktyp) { + mt.Flags |= 8 + } + if hashMightPanic(ktyp) { + mt.Flags |= 16 + } + mt.PtrToThis = 0 + + ti, _ := lookupCache.LoadOrStore(ckey, toRType(&mt.Type)) + return ti.(Type) +} + +func bucketOf(ktyp, etyp *abi.Type) *abi.Type { + if ktyp.Size_ > abi.OldMapMaxKeyBytes { + ktyp = ptrTo(ktyp) + } + if etyp.Size_ > abi.OldMapMaxElemBytes { + etyp = ptrTo(etyp) + } + + // Prepare GC data if any. + // A bucket is at most bucketSize*(1+maxKeySize+maxValSize)+ptrSize bytes, + // or 2064 bytes, or 258 pointer-size words, or 33 bytes of pointer bitmap. + // Note that since the key and value are known to be <= 128 bytes, + // they're guaranteed to have bitmaps instead of GC programs. + var gcdata *byte + var ptrdata uintptr + + size := abi.OldMapBucketCount*(1+ktyp.Size_+etyp.Size_) + goarch.PtrSize + if size&uintptr(ktyp.Align_-1) != 0 || size&uintptr(etyp.Align_-1) != 0 { + panic("reflect: bad size computation in MapOf") + } + + if ktyp.Pointers() || etyp.Pointers() { + nptr := (abi.OldMapBucketCount*(1+ktyp.Size_+etyp.Size_) + goarch.PtrSize) / goarch.PtrSize + n := (nptr + 7) / 8 + + // Runtime needs pointer masks to be a multiple of uintptr in size. + n = (n + goarch.PtrSize - 1) &^ (goarch.PtrSize - 1) + mask := make([]byte, n) + base := uintptr(abi.OldMapBucketCount / goarch.PtrSize) + + if ktyp.Pointers() { + emitGCMask(mask, base, ktyp, abi.OldMapBucketCount) + } + base += abi.OldMapBucketCount * ktyp.Size_ / goarch.PtrSize + + if etyp.Pointers() { + emitGCMask(mask, base, etyp, abi.OldMapBucketCount) + } + base += abi.OldMapBucketCount * etyp.Size_ / goarch.PtrSize + + word := base + mask[word/8] |= 1 << (word % 8) + gcdata = &mask[0] + ptrdata = (word + 1) * goarch.PtrSize + + // overflow word must be last + if ptrdata != size { + panic("reflect: bad layout computation in MapOf") + } + } + + b := &abi.Type{ + Align_: goarch.PtrSize, + Size_: size, + Kind_: abi.Struct, + PtrBytes: ptrdata, + GCData: gcdata, + } + s := "bucket(" + stringFor(ktyp) + "," + stringFor(etyp) + ")" + b.Str = resolveReflectName(newName(s, "", false, false)) + return b +} + +var stringType = rtypeOf("") + +// MapIndex returns the value associated with key in the map v. +// It panics if v's Kind is not [Map]. +// It returns the zero Value if key is not found in the map or if v represents a nil map. +// As in Go, the key's value must be assignable to the map's key type. +func (v Value) MapIndex(key Value) Value { + v.mustBe(Map) + tt := (*mapType)(unsafe.Pointer(v.typ())) + + // Do not require key to be exported, so that DeepEqual + // and other programs can use all the keys returned by + // MapKeys as arguments to MapIndex. If either the map + // or the key is unexported, though, the result will be + // considered unexported. This is consistent with the + // behavior for structs, which allow read but not write + // of unexported fields. + + var e unsafe.Pointer + if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.OldMapMaxElemBytes { + k := *(*string)(key.ptr) + e = mapaccess_faststr(v.typ(), v.pointer(), k) + } else { + key = key.assignTo("reflect.Value.MapIndex", tt.Key, nil) + var k unsafe.Pointer + if key.flag&flagIndir != 0 { + k = key.ptr + } else { + k = unsafe.Pointer(&key.ptr) + } + e = mapaccess(v.typ(), v.pointer(), k) + } + if e == nil { + return Value{} + } + typ := tt.Elem + fl := (v.flag | key.flag).ro() + fl |= flag(typ.Kind()) + return copyVal(typ, fl, e) +} + +// MapKeys returns a slice containing all the keys present in the map, +// in unspecified order. +// It panics if v's Kind is not [Map]. +// It returns an empty slice if v represents a nil map. +func (v Value) MapKeys() []Value { + v.mustBe(Map) + tt := (*mapType)(unsafe.Pointer(v.typ())) + keyType := tt.Key + + fl := v.flag.ro() | flag(keyType.Kind()) + + m := v.pointer() + mlen := int(0) + if m != nil { + mlen = maplen(m) + } + var it hiter + mapiterinit(v.typ(), m, &it) + a := make([]Value, mlen) + var i int + for i = 0; i < len(a); i++ { + key := it.key + if key == nil { + // Someone deleted an entry from the map since we + // called maplen above. It's a data race, but nothing + // we can do about it. + break + } + a[i] = copyVal(keyType, fl, key) + mapiternext(&it) + } + return a[:i] +} + +// hiter's structure matches runtime.hiter's structure. +// Having a clone here allows us to embed a map iterator +// inside type MapIter so that MapIters can be re-used +// without doing any allocations. +type hiter struct { + key unsafe.Pointer + elem unsafe.Pointer + t unsafe.Pointer + h unsafe.Pointer + buckets unsafe.Pointer + bptr unsafe.Pointer + overflow *[]unsafe.Pointer + oldoverflow *[]unsafe.Pointer + startBucket uintptr + offset uint8 + wrapped bool + B uint8 + i uint8 + bucket uintptr + checkBucket uintptr + clearSeq uint64 +} + +func (h *hiter) initialized() bool { + return h.t != nil +} + +// A MapIter is an iterator for ranging over a map. +// See [Value.MapRange]. +type MapIter struct { + m Value + hiter hiter +} + +// Key returns the key of iter's current map entry. +func (iter *MapIter) Key() Value { + if !iter.hiter.initialized() { + panic("MapIter.Key called before Next") + } + iterkey := iter.hiter.key + if iterkey == nil { + panic("MapIter.Key called on exhausted iterator") + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + ktype := t.Key + return copyVal(ktype, iter.m.flag.ro()|flag(ktype.Kind()), iterkey) +} + +// SetIterKey assigns to v the key of iter's current map entry. +// It is equivalent to v.Set(iter.Key()), but it avoids allocating a new Value. +// As in Go, the key must be assignable to v's type and +// must not be derived from an unexported field. +// It panics if [Value.CanSet] returns false. +func (v Value) SetIterKey(iter *MapIter) { + if !iter.hiter.initialized() { + panic("reflect: Value.SetIterKey called before Next") + } + iterkey := iter.hiter.key + if iterkey == nil { + panic("reflect: Value.SetIterKey called on exhausted iterator") + } + + v.mustBeAssignable() + var target unsafe.Pointer + if v.kind() == Interface { + target = v.ptr + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + ktype := t.Key + + iter.m.mustBeExported() // do not let unexported m leak + key := Value{ktype, iterkey, iter.m.flag | flag(ktype.Kind()) | flagIndir} + key = key.assignTo("reflect.MapIter.SetKey", v.typ(), target) + typedmemmove(v.typ(), v.ptr, key.ptr) +} + +// Value returns the value of iter's current map entry. +func (iter *MapIter) Value() Value { + if !iter.hiter.initialized() { + panic("MapIter.Value called before Next") + } + iterelem := iter.hiter.elem + if iterelem == nil { + panic("MapIter.Value called on exhausted iterator") + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + vtype := t.Elem + return copyVal(vtype, iter.m.flag.ro()|flag(vtype.Kind()), iterelem) +} + +// SetIterValue assigns to v the value of iter's current map entry. +// It is equivalent to v.Set(iter.Value()), but it avoids allocating a new Value. +// As in Go, the value must be assignable to v's type and +// must not be derived from an unexported field. +// It panics if [Value.CanSet] returns false. +func (v Value) SetIterValue(iter *MapIter) { + if !iter.hiter.initialized() { + panic("reflect: Value.SetIterValue called before Next") + } + iterelem := iter.hiter.elem + if iterelem == nil { + panic("reflect: Value.SetIterValue called on exhausted iterator") + } + + v.mustBeAssignable() + var target unsafe.Pointer + if v.kind() == Interface { + target = v.ptr + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + vtype := t.Elem + + iter.m.mustBeExported() // do not let unexported m leak + elem := Value{vtype, iterelem, iter.m.flag | flag(vtype.Kind()) | flagIndir} + elem = elem.assignTo("reflect.MapIter.SetValue", v.typ(), target) + typedmemmove(v.typ(), v.ptr, elem.ptr) +} + +// Next advances the map iterator and reports whether there is another +// entry. It returns false when iter is exhausted; subsequent +// calls to [MapIter.Key], [MapIter.Value], or [MapIter.Next] will panic. +func (iter *MapIter) Next() bool { + if !iter.m.IsValid() { + panic("MapIter.Next called on an iterator that does not have an associated map Value") + } + if !iter.hiter.initialized() { + mapiterinit(iter.m.typ(), iter.m.pointer(), &iter.hiter) + } else { + if iter.hiter.key == nil { + panic("MapIter.Next called on exhausted iterator") + } + mapiternext(&iter.hiter) + } + return iter.hiter.key != nil +} + +// Reset modifies iter to iterate over v. +// It panics if v's Kind is not [Map] and v is not the zero Value. +// Reset(Value{}) causes iter to not to refer to any map, +// which may allow the previously iterated-over map to be garbage collected. +func (iter *MapIter) Reset(v Value) { + if v.IsValid() { + v.mustBe(Map) + } + iter.m = v + iter.hiter = hiter{} +} + +// MapRange returns a range iterator for a map. +// It panics if v's Kind is not [Map]. +// +// Call [MapIter.Next] to advance the iterator, and [MapIter.Key]/[MapIter.Value] to access each entry. +// [MapIter.Next] returns false when the iterator is exhausted. +// MapRange follows the same iteration semantics as a range statement. +// +// Example: +// +// iter := reflect.ValueOf(m).MapRange() +// for iter.Next() { +// k := iter.Key() +// v := iter.Value() +// ... +// } +func (v Value) MapRange() *MapIter { + // This is inlinable to take advantage of "function outlining". + // The allocation of MapIter can be stack allocated if the caller + // does not allow it to escape. + // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ + if v.kind() != Map { + v.panicNotMap() + } + return &MapIter{m: v} +} + +// SetMapIndex sets the element associated with key in the map v to elem. +// It panics if v's Kind is not [Map]. +// If elem is the zero Value, SetMapIndex deletes the key from the map. +// Otherwise if v holds a nil map, SetMapIndex will panic. +// As in Go, key's elem must be assignable to the map's key type, +// and elem's value must be assignable to the map's elem type. +func (v Value) SetMapIndex(key, elem Value) { + v.mustBe(Map) + v.mustBeExported() + key.mustBeExported() + tt := (*mapType)(unsafe.Pointer(v.typ())) + + if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.OldMapMaxElemBytes { + k := *(*string)(key.ptr) + if elem.typ() == nil { + mapdelete_faststr(v.typ(), v.pointer(), k) + return + } + elem.mustBeExported() + elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) + var e unsafe.Pointer + if elem.flag&flagIndir != 0 { + e = elem.ptr + } else { + e = unsafe.Pointer(&elem.ptr) + } + mapassign_faststr(v.typ(), v.pointer(), k, e) + return + } + + key = key.assignTo("reflect.Value.SetMapIndex", tt.Key, nil) + var k unsafe.Pointer + if key.flag&flagIndir != 0 { + k = key.ptr + } else { + k = unsafe.Pointer(&key.ptr) + } + if elem.typ() == nil { + mapdelete(v.typ(), v.pointer(), k) + return + } + elem.mustBeExported() + elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) + var e unsafe.Pointer + if elem.flag&flagIndir != 0 { + e = elem.ptr + } else { + e = unsafe.Pointer(&elem.ptr) + } + mapassign(v.typ(), v.pointer(), k, e) +} + +// Force slow panicking path not inlined, so it won't add to the +// inlining budget of the caller. +// TODO: undo when the inliner is no longer bottom-up only. +// +//go:noinline +func (f flag) panicNotMap() { + f.mustBe(Map) +} diff --git a/src/reflect/map_noswiss_test.go b/src/reflect/map_noswiss_test.go new file mode 100644 index 00000000..52fcf895 --- /dev/null +++ b/src/reflect/map_noswiss_test.go @@ -0,0 +1,60 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +package reflect_test + +import ( + "internal/abi" + "internal/goarch" + . "reflect" + "testing" +) + +func testGCBitsMap(t *testing.T) { + const bucketCount = abi.OldMapBucketCount + + hdr := make([]byte, bucketCount/goarch.PtrSize) + + verifyMapBucket := func(t *testing.T, k, e Type, m any, want []byte) { + verifyGCBits(t, MapBucketOf(k, e), want) + verifyGCBits(t, CachedBucketOf(TypeOf(m)), want) + } + verifyMapBucket(t, + Tscalar, Tptr, + map[Xscalar]Xptr(nil), + join(hdr, rep(bucketCount, lit(0)), rep(bucketCount, lit(1)), lit(1))) + verifyMapBucket(t, + Tscalarptr, Tptr, + map[Xscalarptr]Xptr(nil), + join(hdr, rep(bucketCount, lit(0, 1)), rep(bucketCount, lit(1)), lit(1))) + verifyMapBucket(t, Tint64, Tptr, + map[int64]Xptr(nil), + join(hdr, rep(bucketCount, rep(8/goarch.PtrSize, lit(0))), rep(bucketCount, lit(1)), lit(1))) + verifyMapBucket(t, + Tscalar, Tscalar, + map[Xscalar]Xscalar(nil), + empty) + verifyMapBucket(t, + ArrayOf(2, Tscalarptr), ArrayOf(3, Tptrscalar), + map[[2]Xscalarptr][3]Xptrscalar(nil), + join(hdr, rep(bucketCount*2, lit(0, 1)), rep(bucketCount*3, lit(1, 0)), lit(1))) + verifyMapBucket(t, + ArrayOf(64/goarch.PtrSize, Tscalarptr), ArrayOf(64/goarch.PtrSize, Tptrscalar), + map[[64 / goarch.PtrSize]Xscalarptr][64 / goarch.PtrSize]Xptrscalar(nil), + join(hdr, rep(bucketCount*64/goarch.PtrSize, lit(0, 1)), rep(bucketCount*64/goarch.PtrSize, lit(1, 0)), lit(1))) + verifyMapBucket(t, + ArrayOf(64/goarch.PtrSize+1, Tscalarptr), ArrayOf(64/goarch.PtrSize, Tptrscalar), + map[[64/goarch.PtrSize + 1]Xscalarptr][64 / goarch.PtrSize]Xptrscalar(nil), + join(hdr, rep(bucketCount, lit(1)), rep(bucketCount*64/goarch.PtrSize, lit(1, 0)), lit(1))) + verifyMapBucket(t, + ArrayOf(64/goarch.PtrSize, Tscalarptr), ArrayOf(64/goarch.PtrSize+1, Tptrscalar), + map[[64 / goarch.PtrSize]Xscalarptr][64/goarch.PtrSize + 1]Xptrscalar(nil), + join(hdr, rep(bucketCount*64/goarch.PtrSize, lit(0, 1)), rep(bucketCount, lit(1)), lit(1))) + verifyMapBucket(t, + ArrayOf(64/goarch.PtrSize+1, Tscalarptr), ArrayOf(64/goarch.PtrSize+1, Tptrscalar), + map[[64/goarch.PtrSize + 1]Xscalarptr][64/goarch.PtrSize + 1]Xptrscalar(nil), + join(hdr, rep(bucketCount, lit(1)), rep(bucketCount, lit(1)), lit(1))) +} diff --git a/src/reflect/map_swiss.go b/src/reflect/map_swiss.go new file mode 100644 index 00000000..2eac51e5 --- /dev/null +++ b/src/reflect/map_swiss.go @@ -0,0 +1,460 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package reflect + +import ( + "internal/abi" + "internal/race" + "internal/runtime/maps" + "internal/runtime/sys" + "unsafe" +) + +// mapType represents a map type. +// +// TODO(prattmic): Only used within this file, could be cleaned up. +type mapType = abi.SwissMapType + +func (t *rtype) Key() Type { + if t.Kind() != Map { + panic("reflect: Key of non-map type " + t.String()) + } + tt := (*mapType)(unsafe.Pointer(t)) + return toType(tt.Key) +} + +// MapOf returns the map type with the given key and element types. +// For example, if k represents int and e represents string, +// MapOf(k, e) represents map[int]string. +// +// If the key type is not a valid map key type (that is, if it does +// not implement Go's == operator), MapOf panics. +func MapOf(key, elem Type) Type { + ktyp := key.common() + etyp := elem.common() + + if ktyp.Equal == nil { + panic("reflect.MapOf: invalid key type " + stringFor(ktyp)) + } + + // Look in cache. + ckey := cacheKey{Map, ktyp, etyp, 0} + if mt, ok := lookupCache.Load(ckey); ok { + return mt.(Type) + } + + // Look in known types. + s := "map[" + stringFor(ktyp) + "]" + stringFor(etyp) + for _, tt := range typesByString(s) { + mt := (*mapType)(unsafe.Pointer(tt)) + if mt.Key == ktyp && mt.Elem == etyp { + ti, _ := lookupCache.LoadOrStore(ckey, toRType(tt)) + return ti.(Type) + } + } + + group, slot := groupAndSlotOf(key, elem) + + // Make a map type. + // Note: flag values must match those used in the TMAP case + // in ../cmd/compile/internal/reflectdata/reflect.go:writeType. + var imap any = (map[unsafe.Pointer]unsafe.Pointer)(nil) + mt := **(**mapType)(unsafe.Pointer(&imap)) + mt.Str = resolveReflectName(newName(s, "", false, false)) + mt.TFlag = 0 + mt.Hash = fnv1(etyp.Hash, 'm', byte(ktyp.Hash>>24), byte(ktyp.Hash>>16), byte(ktyp.Hash>>8), byte(ktyp.Hash)) + mt.Key = ktyp + mt.Elem = etyp + mt.Group = group.common() + mt.Hasher = func(p unsafe.Pointer, seed uintptr) uintptr { + return typehash(ktyp, p, seed) + } + mt.GroupSize = mt.Group.Size() + mt.SlotSize = slot.Size() + mt.ElemOff = slot.Field(1).Offset + mt.Flags = 0 + if needKeyUpdate(ktyp) { + mt.Flags |= abi.SwissMapNeedKeyUpdate + } + if hashMightPanic(ktyp) { + mt.Flags |= abi.SwissMapHashMightPanic + } + if ktyp.Size_ > abi.SwissMapMaxKeyBytes { + mt.Flags |= abi.SwissMapIndirectKey + } + if etyp.Size_ > abi.SwissMapMaxKeyBytes { + mt.Flags |= abi.SwissMapIndirectElem + } + mt.PtrToThis = 0 + + ti, _ := lookupCache.LoadOrStore(ckey, toRType(&mt.Type)) + return ti.(Type) +} + +func groupAndSlotOf(ktyp, etyp Type) (Type, Type) { + // type group struct { + // ctrl uint64 + // slots [abi.SwissMapGroupSlots]struct { + // key keyType + // elem elemType + // } + // } + + if ktyp.Size() > abi.SwissMapMaxKeyBytes { + ktyp = PointerTo(ktyp) + } + if etyp.Size() > abi.SwissMapMaxElemBytes { + etyp = PointerTo(etyp) + } + + fields := []StructField{ + { + Name: "Key", + Type: ktyp, + }, + { + Name: "Elem", + Type: etyp, + }, + } + slot := StructOf(fields) + + fields = []StructField{ + { + Name: "Ctrl", + Type: TypeFor[uint64](), + }, + { + Name: "Slots", + Type: ArrayOf(abi.SwissMapGroupSlots, slot), + }, + } + group := StructOf(fields) + return group, slot +} + +var stringType = rtypeOf("") + +// MapIndex returns the value associated with key in the map v. +// It panics if v's Kind is not [Map]. +// It returns the zero Value if key is not found in the map or if v represents a nil map. +// As in Go, the key's value must be assignable to the map's key type. +func (v Value) MapIndex(key Value) Value { + v.mustBe(Map) + tt := (*mapType)(unsafe.Pointer(v.typ())) + + // Do not require key to be exported, so that DeepEqual + // and other programs can use all the keys returned by + // MapKeys as arguments to MapIndex. If either the map + // or the key is unexported, though, the result will be + // considered unexported. This is consistent with the + // behavior for structs, which allow read but not write + // of unexported fields. + + var e unsafe.Pointer + if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.SwissMapMaxElemBytes { + k := *(*string)(key.ptr) + e = mapaccess_faststr(v.typ(), v.pointer(), k) + } else { + key = key.assignTo("reflect.Value.MapIndex", tt.Key, nil) + var k unsafe.Pointer + if key.flag&flagIndir != 0 { + k = key.ptr + } else { + k = unsafe.Pointer(&key.ptr) + } + e = mapaccess(v.typ(), v.pointer(), k) + } + if e == nil { + return Value{} + } + typ := tt.Elem + fl := (v.flag | key.flag).ro() + fl |= flag(typ.Kind()) + return copyVal(typ, fl, e) +} + +// Equivalent to runtime.mapIterStart. +// +//go:noinline +func mapIterStart(t *abi.SwissMapType, m *maps.Map, it *maps.Iter) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + race.ReadPC(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(mapIterStart)) + } + + it.Init(t, m) + it.Next() +} + +// Equivalent to runtime.mapIterNext. +// +//go:noinline +func mapIterNext(it *maps.Iter) { + if race.Enabled { + callerpc := sys.GetCallerPC() + race.ReadPC(unsafe.Pointer(it.Map()), callerpc, abi.FuncPCABIInternal(mapIterNext)) + } + + it.Next() +} + +// MapKeys returns a slice containing all the keys present in the map, +// in unspecified order. +// It panics if v's Kind is not [Map]. +// It returns an empty slice if v represents a nil map. +func (v Value) MapKeys() []Value { + v.mustBe(Map) + tt := (*mapType)(unsafe.Pointer(v.typ())) + keyType := tt.Key + + fl := v.flag.ro() | flag(keyType.Kind()) + + // Escape analysis can't see that the map doesn't escape. It sees an + // escape from maps.IterStart, via assignment into it, even though it + // doesn't escape this function. + mptr := abi.NoEscape(v.pointer()) + m := (*maps.Map)(mptr) + mlen := int(0) + if m != nil { + mlen = maplen(mptr) + } + var it maps.Iter + mapIterStart(tt, m, &it) + a := make([]Value, mlen) + var i int + for i = 0; i < len(a); i++ { + key := it.Key() + if key == nil { + // Someone deleted an entry from the map since we + // called maplen above. It's a data race, but nothing + // we can do about it. + break + } + a[i] = copyVal(keyType, fl, key) + mapIterNext(&it) + } + return a[:i] +} + +// A MapIter is an iterator for ranging over a map. +// See [Value.MapRange]. +type MapIter struct { + m Value + hiter maps.Iter +} + +// TODO(prattmic): only for sharing the linkname declarations with old maps. +// Remove with old maps. +type hiter = maps.Iter + +// Key returns the key of iter's current map entry. +func (iter *MapIter) Key() Value { + if !iter.hiter.Initialized() { + panic("MapIter.Key called before Next") + } + iterkey := iter.hiter.Key() + if iterkey == nil { + panic("MapIter.Key called on exhausted iterator") + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + ktype := t.Key + return copyVal(ktype, iter.m.flag.ro()|flag(ktype.Kind()), iterkey) +} + +// SetIterKey assigns to v the key of iter's current map entry. +// It is equivalent to v.Set(iter.Key()), but it avoids allocating a new Value. +// As in Go, the key must be assignable to v's type and +// must not be derived from an unexported field. +// It panics if [Value.CanSet] returns false. +func (v Value) SetIterKey(iter *MapIter) { + if !iter.hiter.Initialized() { + panic("reflect: Value.SetIterKey called before Next") + } + iterkey := iter.hiter.Key() + if iterkey == nil { + panic("reflect: Value.SetIterKey called on exhausted iterator") + } + + v.mustBeAssignable() + var target unsafe.Pointer + if v.kind() == Interface { + target = v.ptr + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + ktype := t.Key + + iter.m.mustBeExported() // do not let unexported m leak + key := Value{ktype, iterkey, iter.m.flag | flag(ktype.Kind()) | flagIndir} + key = key.assignTo("reflect.MapIter.SetKey", v.typ(), target) + typedmemmove(v.typ(), v.ptr, key.ptr) +} + +// Value returns the value of iter's current map entry. +func (iter *MapIter) Value() Value { + if !iter.hiter.Initialized() { + panic("MapIter.Value called before Next") + } + iterelem := iter.hiter.Elem() + if iterelem == nil { + panic("MapIter.Value called on exhausted iterator") + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + vtype := t.Elem + return copyVal(vtype, iter.m.flag.ro()|flag(vtype.Kind()), iterelem) +} + +// SetIterValue assigns to v the value of iter's current map entry. +// It is equivalent to v.Set(iter.Value()), but it avoids allocating a new Value. +// As in Go, the value must be assignable to v's type and +// must not be derived from an unexported field. +// It panics if [Value.CanSet] returns false. +func (v Value) SetIterValue(iter *MapIter) { + if !iter.hiter.Initialized() { + panic("reflect: Value.SetIterValue called before Next") + } + iterelem := iter.hiter.Elem() + if iterelem == nil { + panic("reflect: Value.SetIterValue called on exhausted iterator") + } + + v.mustBeAssignable() + var target unsafe.Pointer + if v.kind() == Interface { + target = v.ptr + } + + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + vtype := t.Elem + + iter.m.mustBeExported() // do not let unexported m leak + elem := Value{vtype, iterelem, iter.m.flag | flag(vtype.Kind()) | flagIndir} + elem = elem.assignTo("reflect.MapIter.SetValue", v.typ(), target) + typedmemmove(v.typ(), v.ptr, elem.ptr) +} + +// Next advances the map iterator and reports whether there is another +// entry. It returns false when iter is exhausted; subsequent +// calls to [MapIter.Key], [MapIter.Value], or [MapIter.Next] will panic. +func (iter *MapIter) Next() bool { + if !iter.m.IsValid() { + panic("MapIter.Next called on an iterator that does not have an associated map Value") + } + if !iter.hiter.Initialized() { + t := (*mapType)(unsafe.Pointer(iter.m.typ())) + m := (*maps.Map)(iter.m.pointer()) + mapIterStart(t, m, &iter.hiter) + } else { + if iter.hiter.Key() == nil { + panic("MapIter.Next called on exhausted iterator") + } + mapIterNext(&iter.hiter) + } + return iter.hiter.Key() != nil +} + +// Reset modifies iter to iterate over v. +// It panics if v's Kind is not [Map] and v is not the zero Value. +// Reset(Value{}) causes iter to not to refer to any map, +// which may allow the previously iterated-over map to be garbage collected. +func (iter *MapIter) Reset(v Value) { + if v.IsValid() { + v.mustBe(Map) + } + iter.m = v + iter.hiter = maps.Iter{} +} + +// MapRange returns a range iterator for a map. +// It panics if v's Kind is not [Map]. +// +// Call [MapIter.Next] to advance the iterator, and [MapIter.Key]/[MapIter.Value] to access each entry. +// [MapIter.Next] returns false when the iterator is exhausted. +// MapRange follows the same iteration semantics as a range statement. +// +// Example: +// +// iter := reflect.ValueOf(m).MapRange() +// for iter.Next() { +// k := iter.Key() +// v := iter.Value() +// ... +// } +func (v Value) MapRange() *MapIter { + // This is inlinable to take advantage of "function outlining". + // The allocation of MapIter can be stack allocated if the caller + // does not allow it to escape. + // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ + if v.kind() != Map { + v.panicNotMap() + } + return &MapIter{m: v} +} + +// SetMapIndex sets the element associated with key in the map v to elem. +// It panics if v's Kind is not [Map]. +// If elem is the zero Value, SetMapIndex deletes the key from the map. +// Otherwise if v holds a nil map, SetMapIndex will panic. +// As in Go, key's elem must be assignable to the map's key type, +// and elem's value must be assignable to the map's elem type. +func (v Value) SetMapIndex(key, elem Value) { + v.mustBe(Map) + v.mustBeExported() + key.mustBeExported() + tt := (*mapType)(unsafe.Pointer(v.typ())) + + if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.SwissMapMaxElemBytes { + k := *(*string)(key.ptr) + if elem.typ() == nil { + mapdelete_faststr(v.typ(), v.pointer(), k) + return + } + elem.mustBeExported() + elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) + var e unsafe.Pointer + if elem.flag&flagIndir != 0 { + e = elem.ptr + } else { + e = unsafe.Pointer(&elem.ptr) + } + mapassign_faststr(v.typ(), v.pointer(), k, e) + return + } + + key = key.assignTo("reflect.Value.SetMapIndex", tt.Key, nil) + var k unsafe.Pointer + if key.flag&flagIndir != 0 { + k = key.ptr + } else { + k = unsafe.Pointer(&key.ptr) + } + if elem.typ() == nil { + mapdelete(v.typ(), v.pointer(), k) + return + } + elem.mustBeExported() + elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) + var e unsafe.Pointer + if elem.flag&flagIndir != 0 { + e = elem.ptr + } else { + e = unsafe.Pointer(&elem.ptr) + } + mapassign(v.typ(), v.pointer(), k, e) +} + +// Force slow panicking path not inlined, so it won't add to the +// inlining budget of the caller. +// TODO: undo when the inliner is no longer bottom-up only. +// +//go:noinline +func (f flag) panicNotMap() { + f.mustBe(Map) +} diff --git a/src/reflect/map_swiss_test.go b/src/reflect/map_swiss_test.go new file mode 100644 index 00000000..621140aa --- /dev/null +++ b/src/reflect/map_swiss_test.go @@ -0,0 +1,30 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package reflect_test + +import ( + "reflect" + "testing" +) + +func testGCBitsMap(t *testing.T) { + // Unlike old maps, we don't manually construct GC data for swiss maps, + // instead using the public reflect API in groupAndSlotOf. +} + +// See also runtime_test.TestGroupSizeZero. +func TestGroupSizeZero(t *testing.T) { + st := reflect.TypeFor[struct{}]() + grp := reflect.MapGroupOf(st, st) + + // internal/runtime/maps when create pointers to slots, even if slots + // are size 0. We should have reserved an extra word to ensure that + // pointers to the zero-size type at the end of group are valid. + if grp.Size() <= 8 { + t.Errorf("Group size got %d want >8", grp.Size()) + } +} diff --git a/src/reflect/type.go b/src/reflect/type.go index 07e2bf16..0e41a6db 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -18,6 +18,7 @@ package reflect import ( "internal/abi" "internal/goarch" + "runtime" "strconv" "sync" "unicode" @@ -262,7 +263,7 @@ type Type interface { /* * These data structures are known to the compiler (../cmd/compile/internal/reflectdata/reflect.go). * A few are known to ../runtime/type.go to convey to debuggers. - * They are also known to ../runtime/type.go. + * They are also known to ../internal/abi/type.go. */ // A Kind represents the specific kind of type that a [Type] represents. @@ -388,11 +389,6 @@ func (t *interfaceType) uncommon() *abi.UncommonType { return t.Uncommon() } -// mapType represents a map type. -type mapType struct { - abi.MapType -} - // ptrType represents a pointer type. type ptrType struct { abi.PtrType @@ -773,14 +769,6 @@ func (t *rtype) FieldByNameFunc(match func(string) bool) (StructField, bool) { return tt.FieldByNameFunc(match) } -func (t *rtype) Key() Type { - if t.Kind() != Map { - panic("reflect: Key of non-map type " + t.String()) - } - tt := (*mapType)(unsafe.Pointer(t)) - return toType(tt.Key) -} - func (t *rtype) Len() int { if t.Kind() != Array { panic("reflect: Len of non-array type " + t.String()) @@ -1127,17 +1115,35 @@ func (t *structType) Field(i int) (f StructField) { } f.Offset = p.Offset - // NOTE(rsc): This is the only allocation in the interface - // presented by a reflect.Type. It would be nice to avoid, - // at least in the common cases, but we need to make sure - // that misbehaving clients of reflect cannot affect other - // uses of reflect. One possibility is CL 5371098, but we - // postponed that ugliness until there is a demonstrated - // need for the performance. This is issue 2320. - f.Index = []int{i} + // We can't safely use this optimization on js or wasi, + // which do not appear to support read-only data. + if i < 256 && runtime.GOOS != "js" && runtime.GOOS != "wasip1" { + staticuint64s := getStaticuint64s() + p := unsafe.Pointer(&(*staticuint64s)[i]) + if unsafe.Sizeof(int(0)) == 4 && goarch.BigEndian { + p = unsafe.Add(p, 4) + } + f.Index = unsafe.Slice((*int)(p), 1) + } else { + // NOTE(rsc): This is the only allocation in the interface + // presented by a reflect.Type. It would be nice to avoid, + // but we need to make sure that misbehaving clients of + // reflect cannot affect other uses of reflect. + // One possibility is CL 5371098, but we postponed that + // ugliness until there is a demonstrated + // need for the performance. This is issue 2320. + f.Index = []int{i} + } return } +// getStaticuint64s returns a pointer to an array of 256 uint64 values, +// defined in the runtime package in read-only memory. +// staticuint64s[0] == 0, staticuint64s[1] == 1, and so forth. +// +//go:linkname getStaticuint64s runtime.getStaticuint64s +func getStaticuint64s() *[256]uint64 + // TODO(gri): Should there be an error/bool indicator if the index // is wrong for FieldByIndex? @@ -1801,79 +1807,6 @@ func ChanOf(dir ChanDir, t Type) Type { return ti.(Type) } -// MapOf returns the map type with the given key and element types. -// For example, if k represents int and e represents string, -// MapOf(k, e) represents map[int]string. -// -// If the key type is not a valid map key type (that is, if it does -// not implement Go's == operator), MapOf panics. -func MapOf(key, elem Type) Type { - ktyp := key.common() - etyp := elem.common() - - if ktyp.Equal == nil { - panic("reflect.MapOf: invalid key type " + stringFor(ktyp)) - } - - // Look in cache. - ckey := cacheKey{Map, ktyp, etyp, 0} - if mt, ok := lookupCache.Load(ckey); ok { - return mt.(Type) - } - - // Look in known types. - s := "map[" + stringFor(ktyp) + "]" + stringFor(etyp) - for _, tt := range typesByString(s) { - mt := (*mapType)(unsafe.Pointer(tt)) - if mt.Key == ktyp && mt.Elem == etyp { - ti, _ := lookupCache.LoadOrStore(ckey, toRType(tt)) - return ti.(Type) - } - } - - // Make a map type. - // Note: flag values must match those used in the TMAP case - // in ../cmd/compile/internal/reflectdata/reflect.go:writeType. - var imap any = (map[unsafe.Pointer]unsafe.Pointer)(nil) - mt := **(**mapType)(unsafe.Pointer(&imap)) - mt.Str = resolveReflectName(newName(s, "", false, false)) - mt.TFlag = 0 - mt.Hash = fnv1(etyp.Hash, 'm', byte(ktyp.Hash>>24), byte(ktyp.Hash>>16), byte(ktyp.Hash>>8), byte(ktyp.Hash)) - mt.Key = ktyp - mt.Elem = etyp - mt.Bucket = bucketOf(ktyp, etyp) - mt.Hasher = func(p unsafe.Pointer, seed uintptr) uintptr { - return typehash(ktyp, p, seed) - } - mt.Flags = 0 - if ktyp.Size_ > abi.MapMaxKeyBytes { - mt.KeySize = uint8(goarch.PtrSize) - mt.Flags |= 1 // indirect key - } else { - mt.KeySize = uint8(ktyp.Size_) - } - if etyp.Size_ > abi.MapMaxElemBytes { - mt.ValueSize = uint8(goarch.PtrSize) - mt.Flags |= 2 // indirect value - } else { - mt.MapType.ValueSize = uint8(etyp.Size_) - } - mt.MapType.BucketSize = uint16(mt.Bucket.Size_) - if isReflexive(ktyp) { - mt.Flags |= 4 - } - if needKeyUpdate(ktyp) { - mt.Flags |= 8 - } - if hashMightPanic(ktyp) { - mt.Flags |= 16 - } - mt.PtrToThis = 0 - - ti, _ := lookupCache.LoadOrStore(ckey, toRType(&mt.Type)) - return ti.(Type) -} - var funcTypes []Type var funcTypesMutex sync.Mutex @@ -2106,79 +2039,9 @@ func hashMightPanic(t *abi.Type) bool { } } -func bucketOf(ktyp, etyp *abi.Type) *abi.Type { - if ktyp.Size_ > abi.MapMaxKeyBytes { - ktyp = ptrTo(ktyp) - } - if etyp.Size_ > abi.MapMaxElemBytes { - etyp = ptrTo(etyp) - } - - // Prepare GC data if any. - // A bucket is at most bucketSize*(1+maxKeySize+maxValSize)+ptrSize bytes, - // or 2064 bytes, or 258 pointer-size words, or 33 bytes of pointer bitmap. - // Note that since the key and value are known to be <= 128 bytes, - // they're guaranteed to have bitmaps instead of GC programs. - var gcdata *byte - var ptrdata uintptr - - size := abi.MapBucketCount*(1+ktyp.Size_+etyp.Size_) + goarch.PtrSize - if size&uintptr(ktyp.Align_-1) != 0 || size&uintptr(etyp.Align_-1) != 0 { - panic("reflect: bad size computation in MapOf") - } - - if ktyp.Pointers() || etyp.Pointers() { - nptr := (abi.MapBucketCount*(1+ktyp.Size_+etyp.Size_) + goarch.PtrSize) / goarch.PtrSize - n := (nptr + 7) / 8 - - // Runtime needs pointer masks to be a multiple of uintptr in size. - n = (n + goarch.PtrSize - 1) &^ (goarch.PtrSize - 1) - mask := make([]byte, n) - base := uintptr(abi.MapBucketCount / goarch.PtrSize) - - if ktyp.Pointers() { - emitGCMask(mask, base, ktyp, abi.MapBucketCount) - } - base += abi.MapBucketCount * ktyp.Size_ / goarch.PtrSize - - if etyp.Pointers() { - emitGCMask(mask, base, etyp, abi.MapBucketCount) - } - base += abi.MapBucketCount * etyp.Size_ / goarch.PtrSize - - word := base - mask[word/8] |= 1 << (word % 8) - gcdata = &mask[0] - ptrdata = (word + 1) * goarch.PtrSize - - // overflow word must be last - if ptrdata != size { - panic("reflect: bad layout computation in MapOf") - } - } - - b := &abi.Type{ - Align_: goarch.PtrSize, - Size_: size, - Kind_: abi.Struct, - PtrBytes: ptrdata, - GCData: gcdata, - } - s := "bucket(" + stringFor(ktyp) + "," + stringFor(etyp) + ")" - b.Str = resolveReflectName(newName(s, "", false, false)) - return b -} - -func (t *rtype) gcSlice(begin, end uintptr) []byte { - return (*[1 << 30]byte)(unsafe.Pointer(t.t.GCData))[begin:end:end] -} - // emitGCMask writes the GC mask for [n]typ into out, starting at bit // offset base. func emitGCMask(out []byte, base uintptr, typ *abi.Type, n uintptr) { - if typ.Kind_&abi.KindGCProg != 0 { - panic("reflect: unexpected GC program") - } ptrs := typ.PtrBytes / goarch.PtrSize words := typ.Size_ / goarch.PtrSize mask := typ.GcSlice(0, (ptrs+7)/8) @@ -2192,32 +2055,6 @@ func emitGCMask(out []byte, base uintptr, typ *abi.Type, n uintptr) { } } -// appendGCProg appends the GC program for the first ptrdata bytes of -// typ to dst and returns the extended slice. -func appendGCProg(dst []byte, typ *abi.Type) []byte { - if typ.Kind_&abi.KindGCProg != 0 { - // Element has GC program; emit one element. - n := uintptr(*(*uint32)(unsafe.Pointer(typ.GCData))) - prog := typ.GcSlice(4, 4+n-1) - return append(dst, prog...) - } - - // Element is small with pointer mask; use as literal bits. - ptrs := typ.PtrBytes / goarch.PtrSize - mask := typ.GcSlice(0, (ptrs+7)/8) - - // Emit 120-bit chunks of full bytes (max is 127 but we avoid using partial bytes). - for ; ptrs > 120; ptrs -= 120 { - dst = append(dst, 120) - dst = append(dst, mask[:15]...) - mask = mask[15:] - } - - dst = append(dst, byte(ptrs)) - dst = append(dst, mask...) - return dst -} - // SliceOf returns the slice type with element type t. // For example, if t represents int, SliceOf(t) represents []int. func SliceOf(t Type) Type { @@ -2356,8 +2193,6 @@ func StructOf(fields []StructField) Type { fs = make([]structField, len(fields)) repr = make([]byte, 0, 64) fset = map[string]struct{}{} // fields' names - - hasGCProg = false // records whether a struct-field type has a GCProg ) lastzero := uintptr(0) @@ -2375,9 +2210,6 @@ func StructOf(fields []StructField) Type { } f, fpkgpath := runtimeStructField(field) ft := f.Typ - if ft.Kind_&abi.KindGCProg != 0 { - hasGCProg = true - } if fpkgpath != "" { if pkgpath == "" { pkgpath = fpkgpath @@ -2648,51 +2480,19 @@ func StructOf(fields []StructField) Type { typ.TFlag |= abi.TFlagUncommon } - if hasGCProg { - lastPtrField := 0 - for i, ft := range fs { - if ft.Typ.Pointers() { - lastPtrField = i - } - } - prog := []byte{0, 0, 0, 0} // will be length of prog - var off uintptr - for i, ft := range fs { - if i > lastPtrField { - // gcprog should not include anything for any field after - // the last field that contains pointer data - break - } - if !ft.Typ.Pointers() { - // Ignore pointerless fields. - continue - } - // Pad to start of this field with zeros. - if ft.Offset > off { - n := (ft.Offset - off) / goarch.PtrSize - prog = append(prog, 0x01, 0x00) // emit a 0 bit - if n > 1 { - prog = append(prog, 0x81) // repeat previous bit - prog = appendVarint(prog, n-1) // n-1 times - } - off = ft.Offset - } - - prog = appendGCProg(prog, ft.Typ) - off += ft.Typ.PtrBytes - } - prog = append(prog, 0) - *(*uint32)(unsafe.Pointer(&prog[0])) = uint32(len(prog) - 4) - typ.Kind_ |= abi.KindGCProg - typ.GCData = &prog[0] - } else { - typ.Kind_ &^= abi.KindGCProg + if typ.PtrBytes == 0 { + typ.GCData = nil + } else if typ.PtrBytes <= abi.MaxPtrmaskBytes*8*goarch.PtrSize { bv := new(bitVector) addTypeBits(bv, 0, &typ.Type) - if len(bv.data) > 0 { - typ.GCData = &bv.data[0] - } + typ.GCData = &bv.data[0] + } else { + // Runtime will build the mask if needed. We just need to allocate + // space to store it. + typ.TFlag |= abi.TFlagGCMaskOnDemand + typ.GCData = (*byte)(unsafe.Pointer(new(uintptr))) } + typ.Equal = nil if comparable { typ.Equal = func(p, q unsafe.Pointer) bool { @@ -2824,6 +2624,8 @@ func ArrayOf(length int, elem Type) Type { array.Size_ = typ.Size_ * uintptr(length) if length > 0 && typ.Pointers() { array.PtrBytes = typ.Size_*uintptr(length-1) + typ.PtrBytes + } else { + array.PtrBytes = 0 } array.Align_ = typ.Align_ array.FieldAlign_ = typ.FieldAlign_ @@ -2831,21 +2633,18 @@ func ArrayOf(length int, elem Type) Type { array.Slice = &(SliceOf(elem).(*rtype).t) switch { - case !typ.Pointers() || array.Size_ == 0: + case array.PtrBytes == 0: // No pointers. array.GCData = nil - array.PtrBytes = 0 case length == 1: // In memory, 1-element array looks just like the element. - array.Kind_ |= typ.Kind_ & abi.KindGCProg + // We share the bitmask with the element type. + array.TFlag |= typ.TFlag & abi.TFlagGCMaskOnDemand array.GCData = typ.GCData - array.PtrBytes = typ.PtrBytes - case typ.Kind_&abi.KindGCProg == 0 && array.Size_ <= abi.MaxPtrmaskBytes*8*goarch.PtrSize: - // Element is small with pointer mask; array is still small. - // Create direct pointer mask by turning each 1 bit in elem - // into length 1 bits in larger mask. + case array.PtrBytes <= abi.MaxPtrmaskBytes*8*goarch.PtrSize: + // Create pointer mask by repeating the element bitmask Len times. n := (array.PtrBytes/goarch.PtrSize + 7) / 8 // Runtime needs pointer masks to be a multiple of uintptr in size. n = (n + goarch.PtrSize - 1) &^ (goarch.PtrSize - 1) @@ -2854,34 +2653,10 @@ func ArrayOf(length int, elem Type) Type { array.GCData = &mask[0] default: - // Create program that emits one element - // and then repeats to make the array. - prog := []byte{0, 0, 0, 0} // will be length of prog - prog = appendGCProg(prog, typ) - // Pad from ptrdata to size. - elemPtrs := typ.PtrBytes / goarch.PtrSize - elemWords := typ.Size_ / goarch.PtrSize - if elemPtrs < elemWords { - // Emit literal 0 bit, then repeat as needed. - prog = append(prog, 0x01, 0x00) - if elemPtrs+1 < elemWords { - prog = append(prog, 0x81) - prog = appendVarint(prog, elemWords-elemPtrs-1) - } - } - // Repeat length-1 times. - if elemWords < 0x80 { - prog = append(prog, byte(elemWords|0x80)) - } else { - prog = append(prog, 0x80) - prog = appendVarint(prog, elemWords) - } - prog = appendVarint(prog, uintptr(length)-1) - prog = append(prog, 0) - *(*uint32)(unsafe.Pointer(&prog[0])) = uint32(len(prog) - 4) - array.Kind_ |= abi.KindGCProg - array.GCData = &prog[0] - array.PtrBytes = array.Size_ // overestimate but ok; must match program + // Runtime will build the mask if needed. We just need to allocate + // space to store it. + array.TFlag |= abi.TFlagGCMaskOnDemand + array.GCData = (*byte)(unsafe.Pointer(new(uintptr))) } etyp := typ diff --git a/src/reflect/value.go b/src/reflect/value.go index 0854371e..ba5b106c 100644 --- a/src/reflect/value.go +++ b/src/reflect/value.go @@ -1779,265 +1779,6 @@ func (v Value) lenNonSlice() int { panic(&ValueError{"reflect.Value.Len", v.kind()}) } -var stringType = rtypeOf("") - -// MapIndex returns the value associated with key in the map v. -// It panics if v's Kind is not [Map]. -// It returns the zero Value if key is not found in the map or if v represents a nil map. -// As in Go, the key's value must be assignable to the map's key type. -func (v Value) MapIndex(key Value) Value { - v.mustBe(Map) - tt := (*mapType)(unsafe.Pointer(v.typ())) - - // Do not require key to be exported, so that DeepEqual - // and other programs can use all the keys returned by - // MapKeys as arguments to MapIndex. If either the map - // or the key is unexported, though, the result will be - // considered unexported. This is consistent with the - // behavior for structs, which allow read but not write - // of unexported fields. - - var e unsafe.Pointer - if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.MapMaxElemBytes { - k := *(*string)(key.ptr) - e = mapaccess_faststr(v.typ(), v.pointer(), k) - } else { - key = key.assignTo("reflect.Value.MapIndex", tt.Key, nil) - var k unsafe.Pointer - if key.flag&flagIndir != 0 { - k = key.ptr - } else { - k = unsafe.Pointer(&key.ptr) - } - e = mapaccess(v.typ(), v.pointer(), k) - } - if e == nil { - return Value{} - } - typ := tt.Elem - fl := (v.flag | key.flag).ro() - fl |= flag(typ.Kind()) - return copyVal(typ, fl, e) -} - -// MapKeys returns a slice containing all the keys present in the map, -// in unspecified order. -// It panics if v's Kind is not [Map]. -// It returns an empty slice if v represents a nil map. -func (v Value) MapKeys() []Value { - v.mustBe(Map) - tt := (*mapType)(unsafe.Pointer(v.typ())) - keyType := tt.Key - - fl := v.flag.ro() | flag(keyType.Kind()) - - m := v.pointer() - mlen := int(0) - if m != nil { - mlen = maplen(m) - } - var it hiter - mapiterinit(v.typ(), m, &it) - a := make([]Value, mlen) - var i int - for i = 0; i < len(a); i++ { - key := mapiterkey(&it) - if key == nil { - // Someone deleted an entry from the map since we - // called maplen above. It's a data race, but nothing - // we can do about it. - break - } - a[i] = copyVal(keyType, fl, key) - mapiternext(&it) - } - return a[:i] -} - -// hiter's structure matches runtime.hiter's structure. -// Having a clone here allows us to embed a map iterator -// inside type MapIter so that MapIters can be re-used -// without doing any allocations. -type hiter struct { - key unsafe.Pointer - elem unsafe.Pointer - t unsafe.Pointer - h unsafe.Pointer - buckets unsafe.Pointer - bptr unsafe.Pointer - overflow *[]unsafe.Pointer - oldoverflow *[]unsafe.Pointer - startBucket uintptr - offset uint8 - wrapped bool - B uint8 - i uint8 - bucket uintptr - checkBucket uintptr -} - -func (h *hiter) initialized() bool { - return h.t != nil -} - -// A MapIter is an iterator for ranging over a map. -// See [Value.MapRange]. -type MapIter struct { - m Value - hiter hiter -} - -// Key returns the key of iter's current map entry. -func (iter *MapIter) Key() Value { - if !iter.hiter.initialized() { - panic("MapIter.Key called before Next") - } - iterkey := mapiterkey(&iter.hiter) - if iterkey == nil { - panic("MapIter.Key called on exhausted iterator") - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - ktype := t.Key - return copyVal(ktype, iter.m.flag.ro()|flag(ktype.Kind()), iterkey) -} - -// SetIterKey assigns to v the key of iter's current map entry. -// It is equivalent to v.Set(iter.Key()), but it avoids allocating a new Value. -// As in Go, the key must be assignable to v's type and -// must not be derived from an unexported field. -func (v Value) SetIterKey(iter *MapIter) { - if !iter.hiter.initialized() { - panic("reflect: Value.SetIterKey called before Next") - } - iterkey := mapiterkey(&iter.hiter) - if iterkey == nil { - panic("reflect: Value.SetIterKey called on exhausted iterator") - } - - v.mustBeAssignable() - var target unsafe.Pointer - if v.kind() == Interface { - target = v.ptr - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - ktype := t.Key - - iter.m.mustBeExported() // do not let unexported m leak - key := Value{ktype, iterkey, iter.m.flag | flag(ktype.Kind()) | flagIndir} - key = key.assignTo("reflect.MapIter.SetKey", v.typ(), target) - typedmemmove(v.typ(), v.ptr, key.ptr) -} - -// Value returns the value of iter's current map entry. -func (iter *MapIter) Value() Value { - if !iter.hiter.initialized() { - panic("MapIter.Value called before Next") - } - iterelem := mapiterelem(&iter.hiter) - if iterelem == nil { - panic("MapIter.Value called on exhausted iterator") - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - vtype := t.Elem - return copyVal(vtype, iter.m.flag.ro()|flag(vtype.Kind()), iterelem) -} - -// SetIterValue assigns to v the value of iter's current map entry. -// It is equivalent to v.Set(iter.Value()), but it avoids allocating a new Value. -// As in Go, the value must be assignable to v's type and -// must not be derived from an unexported field. -func (v Value) SetIterValue(iter *MapIter) { - if !iter.hiter.initialized() { - panic("reflect: Value.SetIterValue called before Next") - } - iterelem := mapiterelem(&iter.hiter) - if iterelem == nil { - panic("reflect: Value.SetIterValue called on exhausted iterator") - } - - v.mustBeAssignable() - var target unsafe.Pointer - if v.kind() == Interface { - target = v.ptr - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - vtype := t.Elem - - iter.m.mustBeExported() // do not let unexported m leak - elem := Value{vtype, iterelem, iter.m.flag | flag(vtype.Kind()) | flagIndir} - elem = elem.assignTo("reflect.MapIter.SetValue", v.typ(), target) - typedmemmove(v.typ(), v.ptr, elem.ptr) -} - -// Next advances the map iterator and reports whether there is another -// entry. It returns false when iter is exhausted; subsequent -// calls to [MapIter.Key], [MapIter.Value], or [MapIter.Next] will panic. -func (iter *MapIter) Next() bool { - if !iter.m.IsValid() { - panic("MapIter.Next called on an iterator that does not have an associated map Value") - } - if !iter.hiter.initialized() { - mapiterinit(iter.m.typ(), iter.m.pointer(), &iter.hiter) - } else { - if mapiterkey(&iter.hiter) == nil { - panic("MapIter.Next called on exhausted iterator") - } - mapiternext(&iter.hiter) - } - return mapiterkey(&iter.hiter) != nil -} - -// Reset modifies iter to iterate over v. -// It panics if v's Kind is not [Map] and v is not the zero Value. -// Reset(Value{}) causes iter to not to refer to any map, -// which may allow the previously iterated-over map to be garbage collected. -func (iter *MapIter) Reset(v Value) { - if v.IsValid() { - v.mustBe(Map) - } - iter.m = v - iter.hiter = hiter{} -} - -// MapRange returns a range iterator for a map. -// It panics if v's Kind is not [Map]. -// -// Call [MapIter.Next] to advance the iterator, and [MapIter.Key]/[MapIter.Value] to access each entry. -// [MapIter.Next] returns false when the iterator is exhausted. -// MapRange follows the same iteration semantics as a range statement. -// -// Example: -// -// iter := reflect.ValueOf(m).MapRange() -// for iter.Next() { -// k := iter.Key() -// v := iter.Value() -// ... -// } -func (v Value) MapRange() *MapIter { - // This is inlinable to take advantage of "function outlining". - // The allocation of MapIter can be stack allocated if the caller - // does not allow it to escape. - // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ - if v.kind() != Map { - v.panicNotMap() - } - return &MapIter{m: v} -} - -// Force slow panicking path not inlined, so it won't add to the -// inlining budget of the caller. -// TODO: undo when the inliner is no longer bottom-up only. -// -//go:noinline -func (f flag) panicNotMap() { - f.mustBe(Map) -} - // copyVal returns a Value containing the map key or value at ptr, // allocating a new variable as needed. func copyVal(typ *abi.Type, fl flag, ptr unsafe.Pointer) Value { @@ -2331,7 +2072,8 @@ func (v Value) SetBool(x bool) { } // SetBytes sets v's underlying value. -// It panics if v's underlying value is not a slice of bytes. +// It panics if v's underlying value is not a slice of bytes +// or if [Value.CanSet] returns false. func (v Value) SetBytes(x []byte) { v.mustBeAssignable() v.mustBe(Slice) @@ -2342,7 +2084,8 @@ func (v Value) SetBytes(x []byte) { } // setRunes sets v's underlying value. -// It panics if v's underlying value is not a slice of runes (int32s). +// It panics if v's underlying value is not a slice of runes (int32s) +// or if [Value.CanSet] returns false. func (v Value) setRunes(x []rune) { v.mustBeAssignable() v.mustBe(Slice) @@ -2353,7 +2096,8 @@ func (v Value) setRunes(x []rune) { } // SetComplex sets v's underlying value to x. -// It panics if v's Kind is not [Complex64] or [Complex128], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Complex64] or [Complex128], +// or if [Value.CanSet] returns false. func (v Value) SetComplex(x complex128) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2367,7 +2111,8 @@ func (v Value) SetComplex(x complex128) { } // SetFloat sets v's underlying value to x. -// It panics if v's Kind is not [Float32] or [Float64], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Float32] or [Float64], +// or if [Value.CanSet] returns false. func (v Value) SetFloat(x float64) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2381,7 +2126,8 @@ func (v Value) SetFloat(x float64) { } // SetInt sets v's underlying value to x. -// It panics if v's Kind is not [Int], [Int8], [Int16], [Int32], or [Int64], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Int], [Int8], [Int16], [Int32], or [Int64], +// or if [Value.CanSet] returns false. func (v Value) SetInt(x int64) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2401,8 +2147,9 @@ func (v Value) SetInt(x int64) { } // SetLen sets v's length to n. -// It panics if v's Kind is not [Slice] or if n is negative or -// greater than the capacity of the slice. +// It panics if v's Kind is not [Slice], or if n is negative or +// greater than the capacity of the slice, +// or if [Value.CanSet] returns false. func (v Value) SetLen(n int) { v.mustBeAssignable() v.mustBe(Slice) @@ -2414,8 +2161,9 @@ func (v Value) SetLen(n int) { } // SetCap sets v's capacity to n. -// It panics if v's Kind is not [Slice] or if n is smaller than the length or -// greater than the capacity of the slice. +// It panics if v's Kind is not [Slice], or if n is smaller than the length or +// greater than the capacity of the slice, +// or if [Value.CanSet] returns false. func (v Value) SetCap(n int) { v.mustBeAssignable() v.mustBe(Slice) @@ -2426,60 +2174,9 @@ func (v Value) SetCap(n int) { s.Cap = n } -// SetMapIndex sets the element associated with key in the map v to elem. -// It panics if v's Kind is not [Map]. -// If elem is the zero Value, SetMapIndex deletes the key from the map. -// Otherwise if v holds a nil map, SetMapIndex will panic. -// As in Go, key's elem must be assignable to the map's key type, -// and elem's value must be assignable to the map's elem type. -func (v Value) SetMapIndex(key, elem Value) { - v.mustBe(Map) - v.mustBeExported() - key.mustBeExported() - tt := (*mapType)(unsafe.Pointer(v.typ())) - - if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.MapMaxElemBytes { - k := *(*string)(key.ptr) - if elem.typ() == nil { - mapdelete_faststr(v.typ(), v.pointer(), k) - return - } - elem.mustBeExported() - elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) - var e unsafe.Pointer - if elem.flag&flagIndir != 0 { - e = elem.ptr - } else { - e = unsafe.Pointer(&elem.ptr) - } - mapassign_faststr(v.typ(), v.pointer(), k, e) - return - } - - key = key.assignTo("reflect.Value.SetMapIndex", tt.Key, nil) - var k unsafe.Pointer - if key.flag&flagIndir != 0 { - k = key.ptr - } else { - k = unsafe.Pointer(&key.ptr) - } - if elem.typ() == nil { - mapdelete(v.typ(), v.pointer(), k) - return - } - elem.mustBeExported() - elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) - var e unsafe.Pointer - if elem.flag&flagIndir != 0 { - e = elem.ptr - } else { - e = unsafe.Pointer(&elem.ptr) - } - mapassign(v.typ(), v.pointer(), k, e) -} - // SetUint sets v's underlying value to x. -// It panics if v's Kind is not [Uint], [Uintptr], [Uint8], [Uint16], [Uint32], or [Uint64], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Uint], [Uintptr], [Uint8], [Uint16], [Uint32], or [Uint64], +// or if [Value.CanSet] returns false. func (v Value) SetUint(x uint64) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2501,7 +2198,8 @@ func (v Value) SetUint(x uint64) { } // SetPointer sets the [unsafe.Pointer] value v to x. -// It panics if v's Kind is not [UnsafePointer]. +// It panics if v's Kind is not [UnsafePointer] +// or if [Value.CanSet] returns false. func (v Value) SetPointer(x unsafe.Pointer) { v.mustBeAssignable() v.mustBe(UnsafePointer) @@ -2677,7 +2375,7 @@ func (v Value) TrySend(x Value) bool { // Type returns v's type. func (v Value) Type() Type { if v.flag != 0 && v.flag&flagMethod == 0 { - return (*rtype)(noescape(unsafe.Pointer(v.typ_))) // inline of toRType(v.typ()), for own inlining in inline test + return (*rtype)(abi.NoEscape(unsafe.Pointer(v.typ_))) // inline of toRType(v.typ()), for own inlining in inline test } return v.typeSlow() } @@ -2869,8 +2567,8 @@ func arrayAt(p unsafe.Pointer, i int, eltSize uintptr, whySafe string) unsafe.Po // another n elements. After Grow(n), at least n elements can be appended // to the slice without another allocation. // -// It panics if v's Kind is not a [Slice] or if n is negative or too large to -// allocate the memory. +// It panics if v's Kind is not a [Slice], or if n is negative or too large to +// allocate the memory, or if [Value.CanSet] returns false. func (v Value) Grow(n int) { v.mustBeAssignable() v.mustBe(Slice) @@ -2958,6 +2656,7 @@ func AppendSlice(s, t Value) Value { // It returns the number of elements copied. // Dst and src each must have kind [Slice] or [Array], and // dst and src must have the same element type. +// It dst is an [Array], it panics if [Value.CanSet] returns false. // // As a special case, src can have kind [String] if the element type of dst is kind [Uint8]. func Copy(dst, src Value) int { @@ -3904,18 +3603,6 @@ func mapdelete(t *abi.Type, m unsafe.Pointer, key unsafe.Pointer) //go:noescape func mapdelete_faststr(t *abi.Type, m unsafe.Pointer, key string) -//go:noescape -func mapiterinit(t *abi.Type, m unsafe.Pointer, it *hiter) - -//go:noescape -func mapiterkey(it *hiter) (key unsafe.Pointer) - -//go:noescape -func mapiterelem(it *hiter) (elem unsafe.Pointer) - -//go:noescape -func mapiternext(it *hiter) - //go:noescape func maplen(m unsafe.Pointer) int @@ -4018,13 +3705,3 @@ func contentEscapes(x unsafe.Pointer) { escapes(*(*any)(x)) // the dereference may not always be safe, but never executed } } - -// This is just a wrapper around abi.NoEscape. The inlining heuristics are -// finnicky and for whatever reason treat the local call to noescape as much -// lower cost with respect to the inliner budget. (That is, replacing calls to -// noescape with abi.NoEscape will cause inlining tests to fail.) -// -//go:nosplit -func noescape(p unsafe.Pointer) unsafe.Pointer { - return abi.NoEscape(p) -} diff --git a/src/regexp/all_test.go b/src/regexp/all_test.go index c9c046b6..ead184d2 100644 --- a/src/regexp/all_test.go +++ b/src/regexp/all_test.go @@ -965,6 +965,21 @@ func TestUnmarshalText(t *testing.T) { if unmarshaled.String() != goodRe[i] { t.Errorf("UnmarshalText returned unexpected value: %s", unmarshaled.String()) } + + buf := make([]byte, 4, 32) + marshalAppend, err := re.AppendText(buf) + if err != nil { + t.Errorf("regexp %#q failed to marshal: %s", re, err) + continue + } + marshalAppend = marshalAppend[4:] + if err := unmarshaled.UnmarshalText(marshalAppend); err != nil { + t.Errorf("regexp %#q failed to unmarshal: %s", re, err) + continue + } + if unmarshaled.String() != goodRe[i] { + t.Errorf("UnmarshalText returned unexpected value: %s", unmarshaled.String()) + } } t.Run("invalid pattern", func(t *testing.T) { re := new(Regexp) diff --git a/src/regexp/onepass.go b/src/regexp/onepass.go index 53cbd958..96e36066 100644 --- a/src/regexp/onepass.go +++ b/src/regexp/onepass.go @@ -465,12 +465,20 @@ func compileOnePass(prog *syntax.Prog) (p *onePassProg) { syntax.EmptyOp(prog.Inst[prog.Start].Arg)&syntax.EmptyBeginText != syntax.EmptyBeginText { return nil } - // every instruction leading to InstMatch must be EmptyEndText + hasAlt := false + for _, inst := range prog.Inst { + if inst.Op == syntax.InstAlt || inst.Op == syntax.InstAltMatch { + hasAlt = true + break + } + } + // If we have alternates, every instruction leading to InstMatch must be EmptyEndText. + // Also, any match on empty text must be $. for _, inst := range prog.Inst { opOut := prog.Inst[inst.Out].Op switch inst.Op { default: - if opOut == syntax.InstMatch { + if opOut == syntax.InstMatch && hasAlt { return nil } case syntax.InstAlt, syntax.InstAltMatch: diff --git a/src/regexp/onepass_test.go b/src/regexp/onepass_test.go index 3f44dc7b..70eba1f5 100644 --- a/src/regexp/onepass_test.go +++ b/src/regexp/onepass_test.go @@ -142,6 +142,7 @@ var onePassTests = []struct { {`^(?:(a)|(?:a*))$`, false}, {`^(?:(?:(?:.(?:$))?))$`, true}, {`^abcd$`, true}, + {`^abcd`, true}, {`^(?:(?:a{0,})*?)$`, false}, {`^(?:(?:a+)*)$`, true}, {`^(?:(?:a|(?:aa)))$`, true}, @@ -154,6 +155,7 @@ var onePassTests = []struct { {`^(?:(?:aa)|a)$`, true}, {`^[a-c]*`, false}, {`^...$`, true}, + {`^...`, true}, {`^(?:a|(?:aa))$`, true}, {`^a((b))c$`, true}, {`^a.[l-nA-Cg-j]?e$`, true}, diff --git a/src/regexp/regexp.go b/src/regexp/regexp.go index e0609942..253415fb 100644 --- a/src/regexp/regexp.go +++ b/src/regexp/regexp.go @@ -1277,14 +1277,22 @@ func (re *Regexp) Split(s string, n int) []string { return strings } -// MarshalText implements [encoding.TextMarshaler]. The output +// AppendText implements [encoding.TextAppender]. The output // matches that of calling the [Regexp.String] method. // // Note that the output is lossy in some cases: This method does not indicate // POSIX regular expressions (i.e. those compiled by calling [CompilePOSIX]), or // those for which the [Regexp.Longest] method has been called. +func (re *Regexp) AppendText(b []byte) ([]byte, error) { + return append(b, re.String()...), nil +} + +// MarshalText implements [encoding.TextMarshaler]. The output +// matches that of calling the [Regexp.AppendText] method. +// +// See [Regexp.AppendText] for more information. func (re *Regexp) MarshalText() ([]byte, error) { - return []byte(re.String()), nil + return re.AppendText(nil) } // UnmarshalText implements [encoding.TextUnmarshaler] by calling diff --git a/src/regexp/syntax/parse.go b/src/regexp/syntax/parse.go index 26242902..ed239daf 100644 --- a/src/regexp/syntax/parse.go +++ b/src/regexp/syntax/parse.go @@ -621,7 +621,7 @@ func (p *parser) factor(sub []*Regexp) []*Regexp { } // Found end of a run with common leading literal string: - // sub[start:i] all begin with str[0:len(str)], but sub[i] + // sub[start:i] all begin with str[:len(str)], but sub[i] // does not even begin with str[0]. // // Factor out common string and append factored expression to out. diff --git a/src/runtime/HACKING.md b/src/runtime/HACKING.md index e1a43ba8..f0c60f3a 100644 --- a/src/runtime/HACKING.md +++ b/src/runtime/HACKING.md @@ -235,7 +235,7 @@ There are three mechanisms for allocating unmanaged memory: objects of the same type. In general, types that are allocated using any of these should be -marked as not in heap by embedding `runtime/internal/sys.NotInHeap`. +marked as not in heap by embedding `internal/runtime/sys.NotInHeap`. Objects that are allocated in unmanaged memory **must not** contain heap pointers unless the following rules are also obeyed: diff --git a/src/runtime/alg.go b/src/runtime/alg.go index bfb9fa1d..07c115f7 100644 --- a/src/runtime/alg.go +++ b/src/runtime/alg.go @@ -8,6 +8,7 @@ import ( "internal/abi" "internal/cpu" "internal/goarch" + "internal/runtime/sys" "unsafe" ) @@ -34,7 +35,7 @@ func memhash128(p unsafe.Pointer, h uintptr) uintptr { //go:nosplit func memhash_varlen(p unsafe.Pointer, h uintptr) uintptr { - ptr := getclosureptr() + ptr := sys.GetClosurePtr() size := *(*uintptr)(unsafe.Pointer(ptr + unsafe.Sizeof(h))) return memhash(p, h, size) } @@ -56,8 +57,6 @@ var useAeshash bool // - github.com/outcaste-io/ristretto // - github.com/puzpuzpuz/xsync/v2 // - github.com/puzpuzpuz/xsync/v3 -// - github.com/segmentio/parquet-go -// - github.com/parquet-go/parquet-go // - github.com/authzed/spicedb // - github.com/pingcap/badger // @@ -67,28 +66,8 @@ var useAeshash bool //go:linkname memhash func memhash(p unsafe.Pointer, h, s uintptr) uintptr -// memhash32 should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/segmentio/parquet-go -// - github.com/parquet-go/parquet-go -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname memhash32 func memhash32(p unsafe.Pointer, h uintptr) uintptr -// memhash64 should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/segmentio/parquet-go -// - github.com/parquet-go/parquet-go -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname memhash64 func memhash64(p unsafe.Pointer, h uintptr) uintptr // strhash should be an internal detail, @@ -97,7 +76,6 @@ func memhash64(p unsafe.Pointer, h uintptr) uintptr // - github.com/aristanetworks/goarista // - github.com/bytedance/sonic // - github.com/bytedance/go-tagexpr/v2 -// - github.com/cloudwego/frugal // - github.com/cloudwego/dynamicgo // - github.com/v2fly/v2ray-core/v5 // diff --git a/src/runtime/arena.go b/src/runtime/arena.go index cd9a9dfa..0ffc74e8 100644 --- a/src/runtime/arena.go +++ b/src/runtime/arena.go @@ -86,8 +86,8 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/math" - "runtime/internal/sys" + "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) @@ -554,13 +554,7 @@ func userArenaHeapBitsSetType(typ *_type, ptr unsafe.Pointer, s *mspan) { base := s.base() h := s.writeUserArenaHeapBits(uintptr(ptr)) - p := typ.GCData // start of 1-bit pointer mask (or GC program) - var gcProgBits uintptr - if typ.Kind_&abi.KindGCProg != 0 { - // Expand gc program, using the object itself for storage. - gcProgBits = runGCProg(addb(p, 4), (*byte)(ptr)) - p = (*byte)(ptr) - } + p := getGCMask(typ) // start of 1-bit pointer mask nb := typ.PtrBytes / goarch.PtrSize for i := uintptr(0); i < nb; i += ptrBits { @@ -585,11 +579,6 @@ func userArenaHeapBitsSetType(typ *_type, ptr unsafe.Pointer, s *mspan) { h = h.pad(s, typ.Size_-typ.PtrBytes) h.flush(s, uintptr(ptr), typ.Size_) - if typ.Kind_&abi.KindGCProg != 0 { - // Zero out temporary ptrmask buffer inside object. - memclrNoHeapPointers(ptr, (gcProgBits+7)/8) - } - // Update the PtrBytes value in the type information. After this // point, the GC will observe the new bitmap. s.largeType.PtrBytes = uintptr(ptr) - base + typ.PtrBytes @@ -798,11 +787,8 @@ func newUserArenaChunk() (unsafe.Pointer, *mspan) { if asanenabled { // TODO(mknyszek): Track individual objects. - rzSize := computeRZlog(span.elemsize) - span.elemsize -= rzSize - span.largeType.Size_ = span.elemsize + // N.B. span.elemsize includes a redzone already. rzStart := span.base() + span.elemsize - span.userArenaChunkFree = makeAddrRange(span.base(), rzStart) asanpoison(unsafe.Pointer(rzStart), span.limit-rzStart) asanunpoison(unsafe.Pointer(span.base()), span.elemsize) } @@ -813,8 +799,8 @@ func newUserArenaChunk() (unsafe.Pointer, *mspan) { throw("newUserArenaChunk called without a P or outside bootstrapping") } // Note cache c only valid while m acquired; see #47302 - if rate != 1 && userArenaChunkBytes < c.nextSample { - c.nextSample -= userArenaChunkBytes + if rate != 1 && int64(userArenaChunkBytes) < c.nextSample { + c.nextSample -= int64(userArenaChunkBytes) } else { profilealloc(mp, unsafe.Pointer(span.base()), userArenaChunkBytes) } @@ -1067,6 +1053,11 @@ func (h *mheap) allocUserArenaChunk() *mspan { s.freeindex = 1 s.allocCount = 1 + // Adjust size to include redzone. + if asanenabled { + s.elemsize -= redZoneSize(s.elemsize) + } + // Account for this new arena chunk memory. gcController.heapInUse.add(int64(userArenaChunkBytes)) gcController.heapReleased.add(-int64(userArenaChunkBytes)) @@ -1088,7 +1079,7 @@ func (h *mheap) allocUserArenaChunk() *mspan { // This must clear the entire heap bitmap so that it's safe // to allocate noscan data without writing anything out. - s.initHeapBits(true) + s.initHeapBits() // Clear the span preemptively. It's an arena chunk, so let's assume // everything is going to be used. diff --git a/src/runtime/asan.go b/src/runtime/asan.go index d79637a3..6fb1d00c 100644 --- a/src/runtime/asan.go +++ b/src/runtime/asan.go @@ -7,19 +7,20 @@ package runtime import ( + "internal/runtime/sys" "unsafe" ) // Public address sanitizer API. func ASanRead(addr unsafe.Pointer, len int) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanread(addr, uintptr(len), sp, pc) } func ASanWrite(addr unsafe.Pointer, len int) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanwrite(addr, uintptr(len), sp, pc) } @@ -32,16 +33,16 @@ const asanenabled = true //go:linkname asanread //go:nosplit func asanread(addr unsafe.Pointer, sz uintptr) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanread(addr, sz, sp, pc) } //go:linkname asanwrite //go:nosplit func asanwrite(addr unsafe.Pointer, sz uintptr) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanwrite(addr, sz, sp, pc) } diff --git a/src/runtime/asm_loong64.s b/src/runtime/asm_loong64.s index c16b27a0..1c5ced45 100644 --- a/src/runtime/asm_loong64.s +++ b/src/runtime/asm_loong64.s @@ -69,6 +69,10 @@ nocgo: // start this M JAL runtime·mstart(SB) + // Prevent dead-code elimination of debugCallV2, which is + // intended to be called by debuggers. + MOVV $runtime·debugCallV2(SB), R0 + MOVV R0, 1(R0) RET @@ -87,9 +91,8 @@ TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 RET // not reached // func cputicks() int64 -TEXT runtime·cputicks(SB),NOSPLIT,$0-8 +TEXT runtime·cputicks(SB),NOSPLIT,$0-8 RDTIMED R0, R4 - MOVV R4, ret+0(FP) RET /* @@ -209,19 +212,19 @@ noswitch: JMP (R4) // func switchToCrashStack0(fn func()) -TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-8 - MOVV fn+0(FP), REGCTXT // context register - MOVV g_m(g), R4 // curm +TEXT runtime·switchToCrashStack0(SB),NOSPLIT,$0-8 + MOVV R4, REGCTXT // context register + MOVV g_m(g), R5 // curm // set g to gcrash MOVV $runtime·gcrash(SB), g // g = &gcrash JAL runtime·save_g(SB) - MOVV R4, g_m(g) // g.m = curm - MOVV g, m_g0(R4) // curm.g0 = g + MOVV R5, g_m(g) // g.m = curm + MOVV g, m_g0(R5) // curm.g0 = g // switch to crashstack - MOVV (g_stack+stack_hi)(g), R4 - ADDV $(-4*8), R4, R3 + MOVV (g_stack+stack_hi)(g), R5 + ADDV $(-4*8), R5, R3 // call target function MOVV 0(REGCTXT), R6 @@ -344,32 +347,65 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \ NO_LOCAL_POINTERS; \ /* copy arguments to stack */ \ MOVV arg+16(FP), R4; \ - MOVWU argsize+24(FP), R5; \ - MOVV R3, R12; \ + MOVWU argsize+24(FP), R5; \ + MOVV R3, R12; \ + MOVV $16, R13; \ ADDV $8, R12; \ - ADDV R12, R5; \ - BEQ R12, R5, 6(PC); \ - MOVBU (R4), R6; \ - ADDV $1, R4; \ - MOVBU R6, (R12); \ - ADDV $1, R12; \ - JMP -5(PC); \ + BLT R5, R13, check8; \ + /* copy 16 bytes a time */ \ + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R16; \ + BEQ R16, copy16_again; \ +loop16:; \ + VMOVQ (R4), V0; \ + ADDV $16, R4; \ + ADDV $-16, R5; \ + VMOVQ V0, (R12); \ + ADDV $16, R12; \ + BGE R5, R13, loop16; \ + JMP check8; \ +copy16_again:; \ + MOVV (R4), R14; \ + MOVV 8(R4), R15; \ + ADDV $16, R4; \ + ADDV $-16, R5; \ + MOVV R14, (R12); \ + MOVV R15, 8(R12); \ + ADDV $16, R12; \ + BGE R5, R13, copy16_again; \ +check8:; \ + /* R13 = 8 */; \ + SRLV $1, R13; \ + BLT R5, R13, 6(PC); \ + /* copy 8 bytes a time */ \ + MOVV (R4), R14; \ + ADDV $8, R4; \ + ADDV $-8, R5; \ + MOVV R14, (R12); \ + ADDV $8, R12; \ + BEQ R5, R0, 7(PC); \ + /* copy 1 byte a time for the rest */ \ + MOVBU (R4), R14; \ + ADDV $1, R4; \ + ADDV $-1, R5; \ + MOVBU R14, (R12); \ + ADDV $1, R12; \ + JMP -6(PC); \ /* set up argument registers */ \ MOVV regArgs+40(FP), R25; \ JAL ·unspillArgs(SB); \ /* call function */ \ - MOVV f+8(FP), REGCTXT; \ + MOVV f+8(FP), REGCTXT; \ MOVV (REGCTXT), R25; \ PCDATA $PCDATA_StackMapIndex, $0; \ JAL (R25); \ /* copy return values back */ \ MOVV regArgs+40(FP), R25; \ - JAL ·spillArgs(SB); \ + JAL ·spillArgs(SB); \ MOVV argtype+0(FP), R7; \ MOVV arg+16(FP), R4; \ MOVWU n+24(FP), R5; \ MOVWU retoffset+28(FP), R6; \ - ADDV $8, R3, R12; \ + ADDV $8, R3, R12; \ ADDV R6, R12; \ ADDV R6, R4; \ SUBVU R6, R5; \ @@ -882,6 +918,229 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVV $64, R29 JMP gcWriteBarrier<>(SB) +DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large" +GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below + +// debugCallV2 is the entry point for debugger-injected function +// calls on running goroutines. It informs the runtime that a +// debug call has been injected and creates a call frame for the +// debugger to fill in. +// +// To inject a function call, a debugger should: +// 1. Check that the goroutine is in state _Grunning and that +// there are at least 280 bytes free on the stack. +// 2. Set SP as SP-8. +// 3. Store the current LR in (SP) (using the SP after step 2). +// 4. Store the current PC in the LR register. +// 5. Write the desired argument frame size at SP-8 +// 6. Save all machine registers so they can be restored later by the debugger. +// 7. Set the PC to debugCallV2 and resume execution. +// +// If the goroutine is in state _Grunnable, then it's not generally +// safe to inject a call because it may return out via other runtime +// operations. Instead, the debugger should unwind the stack to find +// the return to non-runtime code, add a temporary breakpoint there, +// and inject the call once that breakpoint is hit. +// +// If the goroutine is in any other state, it's not safe to inject a call. +// +// This function communicates back to the debugger by setting R19 and +// invoking BREAK to raise a breakpoint signal. Note that the signal PC of +// the signal triggered by the BREAK instruction is the PC where the signal +// is trapped, not the next PC, so to resume execution, the debugger needs +// to set the signal PC to PC+4. See the comments in the implementation for +// the protocol the debugger is expected to follow. InjectDebugCall in the +// runtime tests demonstrates this protocol. +// +// The debugger must ensure that any pointers passed to the function +// obey escape analysis requirements. Specifically, it must not pass +// a stack pointer to an escaping argument. debugCallV2 cannot check +// this invariant. +// +// This is ABIInternal because Go code injects its PC directly into new +// goroutine stacks. +TEXT runtime·debugCallV2(SB),NOSPLIT|NOFRAME,$0-0 + MOVV R1, -272(R3) + ADDV $-272, R3 + + // We can't do anything that might clobber any of these + // registers before this. + MOVV R2, (4*8)(R3) + MOVV R4, (5*8)(R3) + MOVV R5, (6*8)(R3) + MOVV R6, (7*8)(R3) + MOVV R7, (8*8)(R3) + MOVV R8, (9*8)(R3) + MOVV R9, (10*8)(R3) + MOVV R10, (11*8)(R3) + MOVV R11, (12*8)(R3) + MOVV R12, (13*8)(R3) + MOVV R13, (14*8)(R3) + MOVV R14, (15*8)(R3) + MOVV R15, (16*8)(R3) + MOVV R16, (17*8)(R3) + MOVV R17, (18*8)(R3) + MOVV R18, (19*8)(R3) + MOVV R19, (20*8)(R3) + MOVV R20, (21*8)(R3) + MOVV R21, (22*8)(R3) + MOVV g, (23*8)(R3) + MOVV R23, (24*8)(R3) + MOVV R24, (25*8)(R3) + MOVV R25, (26*8)(R3) + MOVV R26, (27*8)(R3) + MOVV R27, (28*8)(R3) + MOVV R28, (29*8)(R3) + MOVV R29, (30*8)(R3) + MOVV R30, (31*8)(R3) + MOVV R31, (32*8)(R3) + + // Perform a safe-point check. + MOVV R1, 8(R3) + CALL runtime·debugCallCheck(SB) + MOVV 16(R3), R30 + BEQ R30, good + + // The safety check failed. Put the reason string at the top + // of the stack. + MOVV R30, 8(R3) + + MOVV 24(R3), R30 + MOVV R30, 16(R3) + + MOVV $8, R19 + BREAK + JMP restore + +good: + // Registers are saved and it's safe to make a call. + // Open up a call frame, moving the stack if necessary. + // + // Once the frame is allocated, this will set R19 to 0 and + // invoke BREAK. The debugger should write the argument + // frame for the call at SP+8, set up argument registers, + // set the LR as the signal PC + 4, set the PC to the function + // to call, set R29 to point to the closure (if a closure call), + // and resume execution. + // + // If the function returns, this will set R19 to 1 and invoke + // BREAK. The debugger can then inspect any return value saved + // on the stack at SP+8 and in registers. To resume execution, + // the debugger should restore the LR from (SP). + // + // If the function panics, this will set R19 to 2 and invoke BREAK. + // The interface{} value of the panic will be at SP+8. The debugger + // can inspect the panic value and resume execution again. +#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \ + MOVV $MAXSIZE, R27; \ + BLT R27, R30, 5(PC); \ + MOVV $NAME(SB), R28; \ + MOVV R28, 8(R3); \ + CALL runtime·debugCallWrap(SB); \ + JMP restore + + MOVV 264(R3), R30 // the argument frame size + DEBUG_CALL_DISPATCH(debugCall32<>, 32) + DEBUG_CALL_DISPATCH(debugCall64<>, 64) + DEBUG_CALL_DISPATCH(debugCall128<>, 128) + DEBUG_CALL_DISPATCH(debugCall256<>, 256) + DEBUG_CALL_DISPATCH(debugCall512<>, 512) + DEBUG_CALL_DISPATCH(debugCall1024<>, 1024) + DEBUG_CALL_DISPATCH(debugCall2048<>, 2048) + DEBUG_CALL_DISPATCH(debugCall4096<>, 4096) + DEBUG_CALL_DISPATCH(debugCall8192<>, 8192) + DEBUG_CALL_DISPATCH(debugCall16384<>, 16384) + DEBUG_CALL_DISPATCH(debugCall32768<>, 32768) + DEBUG_CALL_DISPATCH(debugCall65536<>, 65536) + // The frame size is too large. Report the error. + MOVV $debugCallFrameTooLarge<>(SB), R30 + MOVV R30, 8(R3) + MOVV $20, R30 + MOVV R30, 16(R3) // length of debugCallFrameTooLarge string + MOVV $8, R19 + BREAK + JMP restore + +restore: + // Calls and failures resume here. + // + // Set R19 to 16 and invoke BREAK. The debugger should restore + // all registers except for PC and SP and resume execution. + MOVV $16, R19 + BREAK + // We must not modify flags after this point. + + // Restore pointer-containing registers, which may have been + // modified from the debugger's copy by stack copying. + MOVV (4*8)(R3), R2 + MOVV (5*8)(R3), R4 + MOVV (6*8)(R3), R5 + MOVV (7*8)(R3), R6 + MOVV (8*8)(R3), R7 + MOVV (9*8)(R3), R8 + MOVV (10*8)(R3), R9 + MOVV (11*8)(R3), R10 + MOVV (12*8)(R3), R11 + MOVV (13*8)(R3), R12 + MOVV (14*8)(R3), R13 + MOVV (15*8)(R3), R14 + MOVV (16*8)(R3), R15 + MOVV (17*8)(R3), R16 + MOVV (18*8)(R3), R17 + MOVV (19*8)(R3), R18 + MOVV (20*8)(R3), R19 + MOVV (21*8)(R3), R20 + MOVV (22*8)(R3), R21 + MOVV (23*8)(R3), g + MOVV (24*8)(R3), R23 + MOVV (25*8)(R3), R24 + MOVV (26*8)(R3), R25 + MOVV (27*8)(R3), R26 + MOVV (28*8)(R3), R27 + MOVV (29*8)(R3), R28 + MOVV (30*8)(R3), R29 + MOVV (31*8)(R3), R30 + MOVV (32*8)(R3), R31 + + MOVV 0(R3), R30 + ADDV $280, R3 // Add 8 more bytes, see saveSigContext + MOVV -8(R3), R1 + JMP (R30) + +// runtime.debugCallCheck assumes that functions defined with the +// DEBUG_CALL_FN macro are safe points to inject calls. +#define DEBUG_CALL_FN(NAME,MAXSIZE) \ +TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \ + NO_LOCAL_POINTERS; \ + MOVV $0, R19; \ + BREAK; \ + MOVV $1, R19; \ + BREAK; \ + RET +DEBUG_CALL_FN(debugCall32<>, 32) +DEBUG_CALL_FN(debugCall64<>, 64) +DEBUG_CALL_FN(debugCall128<>, 128) +DEBUG_CALL_FN(debugCall256<>, 256) +DEBUG_CALL_FN(debugCall512<>, 512) +DEBUG_CALL_FN(debugCall1024<>, 1024) +DEBUG_CALL_FN(debugCall2048<>, 2048) +DEBUG_CALL_FN(debugCall4096<>, 4096) +DEBUG_CALL_FN(debugCall8192<>, 8192) +DEBUG_CALL_FN(debugCall16384<>, 16384) +DEBUG_CALL_FN(debugCall32768<>, 32768) +DEBUG_CALL_FN(debugCall65536<>, 65536) + +// func debugCallPanicked(val interface{}) +TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 + // Copy the panic value to the top of stack at SP+8. + MOVV val_type+0(FP), R30 + MOVV R30, 8(R3) + MOVV val_data+8(FP), R30 + MOVV R30, 16(R3) + MOVV $2, R19 + BREAK + RET + // Note: these functions use a special calling convention to save generated code space. // Arguments are passed in registers, but the space for those arguments are allocated // in the caller's stack frame. These stubs write the args into that stack space and diff --git a/src/runtime/asm_riscv64.h b/src/runtime/asm_riscv64.h new file mode 100644 index 00000000..d4deb093 --- /dev/null +++ b/src/runtime/asm_riscv64.h @@ -0,0 +1,12 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Define features that are guaranteed to be supported by setting the GORISCV64 variable. +// If a feature is supported, there's no need to check it at runtime every time. + +#ifdef GORISCV64_rva22u64 +#define hasZba +#define hasZbb +#define hasZbs +#endif diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 491635b1..ef654a3a 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -80,12 +80,11 @@ TEXT setg_gcc<>(SB),NOSPLIT,$0-0 RET // func cputicks() int64 -TEXT runtime·cputicks(SB),NOSPLIT,$0-8 +TEXT runtime·cputicks(SB),NOSPLIT,$0-0 // RDTIME to emulate cpu ticks // RDCYCLE reads counter that is per HART(core) based // according to the riscv manual, see issue 46737 - RDTIME A0 - MOV A0, ret+0(FP) + RDTIME X10 RET // systemstack_switch is a dummy routine that systemstack leaves at the bottom diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index b44a4f7d..69da583a 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -554,5 +554,73 @@ TEXT wasm_pc_f_loop(SB),NOSPLIT,$0 Return +// wasm_pc_f_loop_export is like wasm_pc_f_loop, except that this takes an +// argument (on Wasm stack) that is a PC_F, and the loop stops when we get +// to that PC in a normal return (not unwinding). +// This is for handling an wasmexport function when it needs to switch the +// stack. +TEXT wasm_pc_f_loop_export(SB),NOSPLIT,$0 + Get PAUSE + I32Eqz +outer: + If + // R1 is whether a function return normally (0) or unwinding (1). + // Start with unwinding. + I32Const $1 + Set R1 + loop: + Loop + // Get PC_F & PC_B from -8(SP) + Get SP + I32Const $8 + I32Sub + I32Load16U $2 // PC_F + Tee R2 + + Get R0 + I32Eq + If // PC_F == R0, we're at the stop PC + Get R1 + I32Eqz + // Break if it is a normal return + BrIf outer // actually jump to after the corresponding End + End + + Get SP + I32Const $8 + I32Sub + I32Load16U $0 // PC_B + + Get R2 // PC_F + CallIndirect $0 + Set R1 // save return/unwinding state for next iteration + + Get PAUSE + I32Eqz + BrIf loop + End + End + + I32Const $0 + Set PAUSE + + Return + TEXT wasm_export_lib(SB),NOSPLIT,$0 UNDEF + +TEXT runtime·pause(SB), NOSPLIT, $0-8 + MOVD newsp+0(FP), SP + I32Const $1 + Set PAUSE + RETUNWIND + +// Called if a wasmexport function is called before runtime initialization +TEXT runtime·notInitialized(SB), NOSPLIT, $0 + MOVD $runtime·wasmStack+(m0Stack__size-16-8)(SB), SP + I32Const $0 // entry PC_B + Call runtime·notInitialized1(SB) + Drop + I32Const $0 // entry PC_B + Call runtime·abort(SB) + UNDEF diff --git a/src/runtime/atomic_loong64.s b/src/runtime/atomic_loong64.s index 4818a827..5332d36f 100644 --- a/src/runtime/atomic_loong64.s +++ b/src/runtime/atomic_loong64.s @@ -5,5 +5,5 @@ #include "textflag.h" TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 - DBAR + DBAR $0x1A // StoreStore barrier RET diff --git a/src/runtime/bitcursor_test.go b/src/runtime/bitcursor_test.go new file mode 100644 index 00000000..3a4c7d0f --- /dev/null +++ b/src/runtime/bitcursor_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + . "runtime" + "testing" +) + +func TestBitCursor(t *testing.T) { + ones := [5]byte{0xff, 0xff, 0xff, 0xff, 0xff} + zeros := [5]byte{0, 0, 0, 0, 0} + + for start := uintptr(0); start < 16; start++ { + for end := start + 1; end < 32; end++ { + buf := zeros + NewBitCursor(&buf[0]).Offset(start).Write(&ones[0], end-start) + + for i := uintptr(0); i < uintptr(len(buf)*8); i++ { + bit := buf[i/8] >> (i % 8) & 1 + if bit == 0 && i >= start && i < end { + t.Errorf("bit %d not set in [%d:%d]", i, start, end) + } + if bit == 1 && (i < start || i >= end) { + t.Errorf("bit %d is set outside [%d:%d]", i, start, end) + } + } + } + } + + for start := uintptr(0); start < 16; start++ { + for end := start + 1; end < 32; end++ { + buf := ones + NewBitCursor(&buf[0]).Offset(start).Write(&zeros[0], end-start) + + for i := uintptr(0); i < uintptr(len(buf)*8); i++ { + bit := buf[i/8] >> (i % 8) & 1 + if bit == 1 && i >= start && i < end { + t.Errorf("bit %d not cleared in [%d:%d]", i, start, end) + } + if bit == 0 && (i < start || i >= end) { + t.Errorf("bit %d cleared outside [%d:%d]", i, start, end) + } + } + } + } +} diff --git a/src/runtime/callers_test.go b/src/runtime/callers_test.go index 49a1d5a6..9429442f 100644 --- a/src/runtime/callers_test.go +++ b/src/runtime/callers_test.go @@ -5,8 +5,8 @@ package runtime_test import ( - "reflect" "runtime" + "slices" "strings" "testing" ) @@ -80,7 +80,7 @@ func testCallersEqual(t *testing.T, pcs []uintptr, want []string) { } got = append(got, frame.Function) } - if !reflect.DeepEqual(want, got) { + if !slices.Equal(want, got) { t.Fatalf("wanted %v, got %v", want, got) } } diff --git a/src/runtime/cgo.go b/src/runtime/cgo.go index 8285d87f..eca905ba 100644 --- a/src/runtime/cgo.go +++ b/src/runtime/cgo.go @@ -72,11 +72,20 @@ var cgoHasExtraM bool // cgoUse should not actually be called (see cgoAlwaysFalse). func cgoUse(any) { throw("cgoUse should not be called") } +// cgoKeepAlive is called by cgo-generated code (using go:linkname to get at +// an unexported name). This call keeps its argument alive until the call site; +// cgo emits the call after the last possible use of the argument by C code. +// cgoKeepAlive is marked in the cgo-generated code as //go:noescape, so +// unlike cgoUse it does not force the argument to escape to the heap. +// This is used to implement the #cgo noescape directive. +func cgoKeepAlive(any) { throw("cgoKeepAlive should not be called") } + // cgoAlwaysFalse is a boolean value that is always false. -// The cgo-generated code says if cgoAlwaysFalse { cgoUse(p) }. +// The cgo-generated code says if cgoAlwaysFalse { cgoUse(p) }, +// or if cgoAlwaysFalse { cgoKeepAlive(p) }. // The compiler cannot see that cgoAlwaysFalse is always false, // so it emits the test and keeps the call, giving the desired -// escape analysis result. The test is cheaper than the call. +// escape/alive analysis result. The test is cheaper than the call. var cgoAlwaysFalse bool var cgo_yield = &_cgo_yield diff --git a/src/runtime/cgo/cgo.go b/src/runtime/cgo/cgo.go index 1e3a5029..c37135fb 100644 --- a/src/runtime/cgo/cgo.go +++ b/src/runtime/cgo/cgo.go @@ -32,7 +32,7 @@ package cgo */ import "C" -import "runtime/internal/sys" +import "internal/runtime/sys" // Incomplete is used specifically for the semantics of incomplete C types. type Incomplete struct { diff --git a/src/runtime/cgo/gcc_darwin_amd64.c b/src/runtime/cgo/gcc_darwin_amd64.c index 5b5e369f..1ffbbd75 100644 --- a/src/runtime/cgo/gcc_darwin_amd64.c +++ b/src/runtime/cgo/gcc_darwin_amd64.c @@ -34,6 +34,7 @@ _cgo_sys_thread_start(ThreadStart *ts) size = pthread_get_stacksize_np(pthread_self()); pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, size); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); diff --git a/src/runtime/cgo/gcc_darwin_arm64.c b/src/runtime/cgo/gcc_darwin_arm64.c index f1344de8..4b346929 100644 --- a/src/runtime/cgo/gcc_darwin_arm64.c +++ b/src/runtime/cgo/gcc_darwin_arm64.c @@ -39,6 +39,7 @@ _cgo_sys_thread_start(ThreadStart *ts) size = pthread_get_stacksize_np(pthread_self()); pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, size); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); diff --git a/src/runtime/cgo/gcc_dragonfly_amd64.c b/src/runtime/cgo/gcc_dragonfly_amd64.c index 009d4b4f..646e9e65 100644 --- a/src/runtime/cgo/gcc_dragonfly_amd64.c +++ b/src/runtime/cgo/gcc_dragonfly_amd64.c @@ -33,6 +33,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. diff --git a/src/runtime/cgo/gcc_freebsd.c b/src/runtime/cgo/gcc_freebsd.c index a9412113..1cc582a3 100644 --- a/src/runtime/cgo/gcc_freebsd.c +++ b/src/runtime/cgo/gcc_freebsd.c @@ -45,6 +45,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_freebsd_amd64.c b/src/runtime/cgo/gcc_freebsd_amd64.c index 31905f2a..ec23a00f 100644 --- a/src/runtime/cgo/gcc_freebsd_amd64.c +++ b/src/runtime/cgo/gcc_freebsd_amd64.c @@ -43,6 +43,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_libinit.c b/src/runtime/cgo/gcc_libinit.c index 33a9ff93..2fe76e43 100644 --- a/src/runtime/cgo/gcc_libinit.c +++ b/src/runtime/cgo/gcc_libinit.c @@ -37,8 +37,12 @@ static void (*cgo_context_function)(struct context_arg*); void x_cgo_sys_thread_create(void* (*func)(void*), void* arg) { + pthread_attr_t attr; pthread_t p; - int err = _cgo_try_pthread_create(&p, NULL, func, arg); + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + int err = _cgo_try_pthread_create(&p, &attr, func, arg); if (err != 0) { fprintf(stderr, "pthread_create failed: %s", strerror(err)); abort(); @@ -153,7 +157,6 @@ _cgo_try_pthread_create(pthread_t* thread, const pthread_attr_t* attr, void* (*p for (tries = 0; tries < 20; tries++) { err = pthread_create(thread, attr, pfn, arg); if (err == 0) { - pthread_detach(*thread); return 0; } if (err != EAGAIN) { diff --git a/src/runtime/cgo/gcc_libinit_windows.c b/src/runtime/cgo/gcc_libinit_windows.c index 9a8c65ea..ddc0ad70 100644 --- a/src/runtime/cgo/gcc_libinit_windows.c +++ b/src/runtime/cgo/gcc_libinit_windows.c @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +# ifdef __CYGWIN__ +#error "don't use the cygwin compiler to build native Windows programs; use MinGW instead" +#else +// Exclude the following code from Cygwin builds. +// Cygwin doesn't implement process.h nor does it support _beginthread. + #define WIN32_LEAN_AND_MEAN #include #include @@ -156,3 +162,5 @@ void _cgo_beginthread(void (*func)(void*), void* arg) { fprintf(stderr, "runtime: failed to create new OS thread (%d)\n", errno); abort(); } + +#endif // __CYGWIN__ \ No newline at end of file diff --git a/src/runtime/cgo/gcc_linux.c b/src/runtime/cgo/gcc_linux.c index 9624df59..fbdb5e65 100644 --- a/src/runtime/cgo/gcc_linux.c +++ b/src/runtime/cgo/gcc_linux.c @@ -40,6 +40,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_linux_amd64.c b/src/runtime/cgo/gcc_linux_amd64.c index dcb596e2..c81fde28 100644 --- a/src/runtime/cgo/gcc_linux_amd64.c +++ b/src/runtime/cgo/gcc_linux_amd64.c @@ -63,6 +63,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_linux_arm64.c b/src/runtime/cgo/gcc_linux_arm64.c index 0dcff2c0..ce46804b 100644 --- a/src/runtime/cgo/gcc_linux_arm64.c +++ b/src/runtime/cgo/gcc_linux_arm64.c @@ -28,6 +28,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_linux_s390x.c b/src/runtime/cgo/gcc_linux_s390x.c index 4b9f76c3..1fae4e7d 100644 --- a/src/runtime/cgo/gcc_linux_s390x.c +++ b/src/runtime/cgo/gcc_linux_s390x.c @@ -33,6 +33,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_netbsd.c b/src/runtime/cgo/gcc_netbsd.c index 16819ce8..98b629e8 100644 --- a/src/runtime/cgo/gcc_netbsd.c +++ b/src/runtime/cgo/gcc_netbsd.c @@ -35,6 +35,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_openbsd.c b/src/runtime/cgo/gcc_openbsd.c index 3a4e5454..b31c312f 100644 --- a/src/runtime/cgo/gcc_openbsd.c +++ b/src/runtime/cgo/gcc_openbsd.c @@ -34,6 +34,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. diff --git a/src/runtime/cgo/gcc_ppc64x.c b/src/runtime/cgo/gcc_ppc64x.c index 98a65495..e1a5c14f 100644 --- a/src/runtime/cgo/gcc_ppc64x.c +++ b/src/runtime/cgo/gcc_ppc64x.c @@ -35,6 +35,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_stack_unix.c b/src/runtime/cgo/gcc_stack_unix.c index df0049a4..9550cd78 100644 --- a/src/runtime/cgo/gcc_stack_unix.c +++ b/src/runtime/cgo/gcc_stack_unix.c @@ -21,7 +21,7 @@ x_cgo_getstackbound(uintptr bounds[2]) // Needed before pthread_getattr_np, too, since before glibc 2.32 // it did not call pthread_attr_init in all cases (see #65625). pthread_attr_init(&attr); -#if defined(__GLIBC__) || (defined(__sun) && !defined(__illumos__)) +#if defined(__GLIBC__) || defined(__BIONIC__) || (defined(__sun) && !defined(__illumos__)) // pthread_getattr_np is a GNU extension supported in glibc. // Solaris is not glibc but does support pthread_getattr_np // (and the fallback doesn't work...). Illumos does not. diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 972de4fe..326674cd 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -88,7 +88,7 @@ import ( "internal/abi" "internal/goarch" "internal/goexperiment" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -425,6 +425,13 @@ func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { restore := true defer unwindm(&restore) + var ditAlreadySet bool + if debug.dataindependenttiming == 1 && gp.m.isextra { + // We only need to enable DIT for threads that were created by C, as it + // should already by enabled on threads that were created by Go. + ditAlreadySet = sys.EnableDIT() + } + if raceenabled { raceacquire(unsafe.Pointer(&racecgosync)) } @@ -440,6 +447,11 @@ func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { racereleasemerge(unsafe.Pointer(&racecgosync)) } + if debug.dataindependenttiming == 1 && !ditAlreadySet { + // Only unset DIT if it wasn't already enabled when cgocallback was called. + sys.DisableDIT() + } + // Do not unwind m->g0->sched.sp. // Our caller, cgocallback, will do that. restore = false @@ -558,6 +570,17 @@ func cgoCheckPointer(ptr any, arg any) { ep = aep t = ep._type top = false + case abi.Pointer: + // The Go code is indexing into a pointer to an array, + // and we have been passed the pointer-to-array. + // Check the array rather than the pointer. + pt := (*abi.PtrType)(unsafe.Pointer(aep._type)) + t = pt.Elem + if t.Kind_&abi.KindMask != abi.Array { + throw("can't happen") + } + ep = aep + top = false default: throw("can't happen") } diff --git a/src/runtime/cgocheck.go b/src/runtime/cgocheck.go index 3f2c2719..ab804a5a 100644 --- a/src/runtime/cgocheck.go +++ b/src/runtime/cgocheck.go @@ -8,7 +8,6 @@ package runtime import ( - "internal/abi" "internal/goarch" "unsafe" ) @@ -142,52 +141,7 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) { size = ptrdataSize } - if typ.Kind_&abi.KindGCProg == 0 { - cgoCheckBits(src, typ.GCData, off, size) - return - } - - // The type has a GC program. Try to find GC bits somewhere else. - for _, datap := range activeModules() { - if cgoInRange(src, datap.data, datap.edata) { - doff := uintptr(src) - datap.data - cgoCheckBits(add(src, -doff), datap.gcdatamask.bytedata, off+doff, size) - return - } - if cgoInRange(src, datap.bss, datap.ebss) { - boff := uintptr(src) - datap.bss - cgoCheckBits(add(src, -boff), datap.gcbssmask.bytedata, off+boff, size) - return - } - } - - s := spanOfUnchecked(uintptr(src)) - if s.state.get() == mSpanManual { - // There are no heap bits for value stored on the stack. - // For a channel receive src might be on the stack of some - // other goroutine, so we can't unwind the stack even if - // we wanted to. - // We can't expand the GC program without extra storage - // space we can't easily get. - // Fortunately we have the type information. - systemstack(func() { - cgoCheckUsingType(typ, src, off, size) - }) - return - } - - // src must be in the regular heap. - tp := s.typePointersOf(uintptr(src), size) - for { - var addr uintptr - if tp, addr = tp.next(uintptr(src) + size); addr == 0 { - break - } - v := *(*unsafe.Pointer)(unsafe.Pointer(addr)) - if cgoIsGoPointer(v) && !isPinned(v) { - throw(cgoWriteBarrierFail) - } - } + cgoCheckBits(src, getGCMask(typ), off, size) } // cgoCheckBits checks the block of memory at src, for up to size @@ -245,48 +199,5 @@ func cgoCheckUsingType(typ *_type, src unsafe.Pointer, off, size uintptr) { size = ptrdataSize } - if typ.Kind_&abi.KindGCProg == 0 { - cgoCheckBits(src, typ.GCData, off, size) - return - } - switch typ.Kind_ & abi.KindMask { - default: - throw("can't happen") - case abi.Array: - at := (*arraytype)(unsafe.Pointer(typ)) - for i := uintptr(0); i < at.Len; i++ { - if off < at.Elem.Size_ { - cgoCheckUsingType(at.Elem, src, off, size) - } - src = add(src, at.Elem.Size_) - skipped := off - if skipped > at.Elem.Size_ { - skipped = at.Elem.Size_ - } - checked := at.Elem.Size_ - skipped - off -= skipped - if size <= checked { - return - } - size -= checked - } - case abi.Struct: - st := (*structtype)(unsafe.Pointer(typ)) - for _, f := range st.Fields { - if off < f.Typ.Size_ { - cgoCheckUsingType(f.Typ, src, off, size) - } - src = add(src, f.Typ.Size_) - skipped := off - if skipped > f.Typ.Size_ { - skipped = f.Typ.Size_ - } - checked := f.Typ.Size_ - skipped - off -= skipped - if size <= checked { - return - } - size -= checked - } - } + cgoCheckBits(src, getGCMask(typ), off, size) } diff --git a/src/runtime/chan.go b/src/runtime/chan.go index f1cd74a3..8e096537 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -20,7 +20,8 @@ package runtime import ( "internal/abi" "internal/runtime/atomic" - "runtime/internal/math" + "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) @@ -35,6 +36,7 @@ type hchan struct { dataqsiz uint // size of the circular queue buf unsafe.Pointer // points to an array of dataqsiz elements elemsize uint16 + synctest bool // true if created in a synctest bubble closed uint32 timer *timer // timer feeding this chan elemtype *_type // element type @@ -111,6 +113,9 @@ func makechan(t *chantype, size int) *hchan { c.elemsize = uint16(elem.Size_) c.elemtype = elem c.dataqsiz = uint(size) + if getg().syncGroup != nil { + c.synctest = true + } lockInit(&c.lock, lockRankHchan) if debugChan { @@ -153,7 +158,7 @@ func full(c *hchan) bool { // //go:nosplit func chansend1(c *hchan, elem unsafe.Pointer) { - chansend(c, elem, true, getcallerpc()) + chansend(c, elem, true, sys.GetCallerPC()) } /* @@ -185,6 +190,10 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { racereadpc(c.raceaddr(), callerpc, abi.FuncPCABIInternal(chansend)) } + if c.synctest && getg().syncGroup == nil { + panic(plainError("send on synctest channel from outside bubble")) + } + // Fast path: check for failed non-blocking operation without acquiring the lock. // // After observing that the channel is not closed, we observe that the channel is @@ -267,7 +276,11 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { // changes and when we set gp.activeStackChans is not safe for // stack shrinking. gp.parkingOnChan.Store(true) - gopark(chanparkcommit, unsafe.Pointer(&c.lock), waitReasonChanSend, traceBlockChanSend, 2) + reason := waitReasonChanSend + if c.synctest { + reason = waitReasonSynctestChanSend + } + gopark(chanparkcommit, unsafe.Pointer(&c.lock), reason, traceBlockChanSend, 2) // Ensure the value being sent is kept alive until the // receiver copies it out. The sudog has a pointer to the // stack object, but sudogs aren't considered as roots of the @@ -303,6 +316,10 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { // sg must already be dequeued from c. // ep must be non-nil and point to the heap or the caller's stack. func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { + if c.synctest && sg.g.syncGroup != getg().syncGroup { + unlockf() + panic(plainError("send on synctest channel from outside bubble")) + } if raceenabled { if c.dataqsiz == 0 { racesync(c, sg) @@ -406,7 +423,7 @@ func closechan(c *hchan) { } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(c.raceaddr(), callerpc, abi.FuncPCABIInternal(closechan)) racerelease(c.raceaddr()) } @@ -517,6 +534,10 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) throw("unreachable") } + if c.synctest && getg().syncGroup == nil { + panic(plainError("receive on synctest channel from outside bubble")) + } + if c.timer != nil { c.timer.maybeRunChan() } @@ -636,7 +657,11 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) // changes and when we set gp.activeStackChans is not safe for // stack shrinking. gp.parkingOnChan.Store(true) - gopark(chanparkcommit, unsafe.Pointer(&c.lock), waitReasonChanReceive, traceBlockChanRecv, 2) + reason := waitReasonChanReceive + if c.synctest { + reason = waitReasonSynctestChanReceive + } + gopark(chanparkcommit, unsafe.Pointer(&c.lock), reason, traceBlockChanRecv, 2) // someone woke us up if mysg != gp.waiting { @@ -672,6 +697,10 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) // sg must already be dequeued from c. // A non-nil ep must point to the heap or the caller's stack. func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { + if c.synctest && sg.g.syncGroup != getg().syncGroup { + unlockf() + panic(plainError("receive on synctest channel from outside bubble")) + } if c.dataqsiz == 0 { if raceenabled { racesync(c, sg) @@ -750,7 +779,7 @@ func chanparkcommit(gp *g, chanLock unsafe.Pointer) bool { // ... bar // } func selectnbsend(c *hchan, elem unsafe.Pointer) (selected bool) { - return chansend(c, elem, false, getcallerpc()) + return chansend(c, elem, false, sys.GetCallerPC()) } // compiler implements @@ -775,7 +804,7 @@ func selectnbrecv(elem unsafe.Pointer, c *hchan) (selected, received bool) { //go:linkname reflect_chansend reflect.chansend0 func reflect_chansend(c *hchan, elem unsafe.Pointer, nb bool) (selected bool) { - return chansend(c, elem, !nb, getcallerpc()) + return chansend(c, elem, !nb, sys.GetCallerPC()) } //go:linkname reflect_chanrecv reflect.chanrecv @@ -875,8 +904,11 @@ func (q *waitq) dequeue() *sudog { // We use a flag in the G struct to tell us when someone // else has won the race to signal this goroutine but the goroutine // hasn't removed itself from the queue yet. - if sgp.isSelect && !sgp.g.selectDone.CompareAndSwap(0, 1) { - continue + if sgp.isSelect { + if !sgp.g.selectDone.CompareAndSwap(0, 1) { + // We lost the race to wake this goroutine. + continue + } } return sgp diff --git a/src/runtime/coro.go b/src/runtime/coro.go index d93817f9..f2eb8c98 100644 --- a/src/runtime/coro.go +++ b/src/runtime/coro.go @@ -4,7 +4,10 @@ package runtime -import "unsafe" +import ( + "internal/runtime/sys" + "unsafe" +) // A coro represents extra concurrency without extra parallelism, // as would be needed for a coroutine implementation. @@ -39,7 +42,7 @@ type coro struct { func newcoro(f func(*coro)) *coro { c := new(coro) c.f = f - pc := getcallerpc() + pc := sys.GetCallerPC() gp := getg() systemstack(func() { mp := gp.m @@ -134,6 +137,16 @@ func coroswitch_m(gp *g) { // emitting an event for every single transition. trace := traceAcquire() + canCAS := true + sg := gp.syncGroup + if sg != nil { + // If we're in a synctest group, always use casgstatus (which tracks + // group idleness) rather than directly CASing. Mark the group as active + // while we're in the process of transferring control. + canCAS = false + sg.incActive() + } + if locked { // Detach the goroutine from the thread; we'll attach to the goroutine we're // switching to before returning. @@ -152,7 +165,7 @@ func coroswitch_m(gp *g) { // If we can CAS ourselves directly from running to waiting, so do, // keeping the control transfer as lightweight as possible. gp.waitreason = waitReasonCoroutine - if !gp.atomicstatus.CompareAndSwap(_Grunning, _Gwaiting) { + if !canCAS || !gp.atomicstatus.CompareAndSwap(_Grunning, _Gwaiting) { // The CAS failed: use casgstatus, which will take care of // coordinating with the garbage collector about the state change. casgstatus(gp, _Grunning, _Gwaiting) @@ -220,7 +233,7 @@ func coroswitch_m(gp *g) { tryRecordGoroutineProfile(gnext, nil, osyield) } - if !gnext.atomicstatus.CompareAndSwap(_Gwaiting, _Grunning) { + if !canCAS || !gnext.atomicstatus.CompareAndSwap(_Gwaiting, _Grunning) { // The CAS failed: use casgstatus, which will take care of // coordinating with the garbage collector about the state change. casgstatus(gnext, _Gwaiting, _Grunnable) @@ -238,6 +251,10 @@ func coroswitch_m(gp *g) { traceRelease(trace) } + if sg != nil { + sg.decActive() + } + // Switch to gnext. Does not return. gogo(&gnext.sched) } diff --git a/src/runtime/coverage/coverage.go b/src/runtime/coverage/coverage.go index 6b99a0bc..c9f725e3 100644 --- a/src/runtime/coverage/coverage.go +++ b/src/runtime/coverage/coverage.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Package coverage contains APIs for writing coverage profile data at runtime +// from long-running and/or server programs that do not terminate via [os.Exit]. package coverage import ( diff --git a/src/runtime/cpuflags.go b/src/runtime/cpuflags.go index bbe93c5b..e81e50f5 100644 --- a/src/runtime/cpuflags.go +++ b/src/runtime/cpuflags.go @@ -19,6 +19,8 @@ const ( offsetARMHasIDIVA = unsafe.Offsetof(cpu.ARM.HasIDIVA) offsetMIPS64XHasMSA = unsafe.Offsetof(cpu.MIPS64X.HasMSA) + + offsetLOONG64HasLSX = unsafe.Offsetof(cpu.Loong64.HasLSX) ) var ( @@ -31,4 +33,8 @@ var ( armHasVFPv4 bool arm64HasATOMICS bool + + loong64HasLAMCAS bool + loong64HasLAM_BH bool + loong64HasLSX bool ) diff --git a/src/runtime/cpuflags_amd64.go b/src/runtime/cpuflags_amd64.go index 8cca4bca..b6d8c6c1 100644 --- a/src/runtime/cpuflags_amd64.go +++ b/src/runtime/cpuflags_amd64.go @@ -8,17 +8,31 @@ import ( "internal/cpu" ) -var useAVXmemmove bool +var memmoveBits uint8 + +const ( + // avxSupported indicates that the CPU supports AVX instructions. + avxSupported = 1 << 0 + + // repmovsPreferred indicates that REP MOVSx instruction is more + // efficient on the CPU. + repmovsPreferred = 1 << 1 +) func init() { - // Let's remove stepping and reserved fields - processor := processorVersionInfo & 0x0FFF3FF0 - - isIntelBridgeFamily := isIntel && - processor == 0x206A0 || - processor == 0x206D0 || - processor == 0x306A0 || - processor == 0x306E0 - - useAVXmemmove = cpu.X86.HasAVX && !isIntelBridgeFamily + // Here we assume that on modern CPUs with both FSRM and ERMS features, + // copying data blocks of 2KB or larger using the REP MOVSB instruction + // will be more efficient to avoid having to keep up with CPU generations. + // Therefore, we may retain a BlockList mechanism to ensure that microarchitectures + // that do not fit this case may appear in the future. + // We enable it on Intel CPUs first, and we may support more platforms + // in the future. + isERMSNiceCPU := isIntel + useREPMOV := isERMSNiceCPU && cpu.X86.HasERMS && cpu.X86.HasFSRM + if cpu.X86.HasAVX { + memmoveBits |= avxSupported + } + if useREPMOV { + memmoveBits |= repmovsPreferred + } } diff --git a/src/runtime/cpuprof.go b/src/runtime/cpuprof.go index 100a7825..ea4d3a8c 100644 --- a/src/runtime/cpuprof.go +++ b/src/runtime/cpuprof.go @@ -14,7 +14,7 @@ package runtime import ( "internal/abi" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index f9fbdd8f..e9b449ab 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -65,10 +65,6 @@ func TestCgoCallbackGC(t *testing.T) { t.Skip("too slow for mips64x builders") } } - if testenv.Builder() == "darwin-amd64-10_14" { - // TODO(#23011): When the 10.14 builders are gone, remove this skip. - t.Skip("skipping due to platform bug on macOS 10.14; see https://golang.org/issue/43926") - } got := runTestProg(t, "testprogcgo", "CgoCallbackGC") want := "OK\n" if got != want { @@ -754,7 +750,6 @@ func TestNeedmDeadlock(t *testing.T) { } func TestCgoNoCallback(t *testing.T) { - t.Skip("TODO(#56378): enable in Go 1.23") got := runTestProg(t, "testprogcgo", "CgoNoCallback") want := "function marked with #cgo nocallback called back into Go" if !strings.Contains(got, want) { @@ -763,7 +758,6 @@ func TestCgoNoCallback(t *testing.T) { } func TestCgoNoEscape(t *testing.T) { - t.Skip("TODO(#56378): enable in Go 1.23") got := runTestProg(t, "testprogcgo", "CgoNoEscape") want := "OK\n" if got != want { @@ -771,6 +765,15 @@ func TestCgoNoEscape(t *testing.T) { } } +// Issue #63739. +func TestCgoEscapeWithMultiplePointers(t *testing.T) { + got := runTestProg(t, "testprogcgo", "CgoEscapeWithMultiplePointers") + want := "OK\n" + if got != want { + t.Fatalf("output is %s; want %s", got, want) + } +} + func TestCgoTracebackGoroutineProfile(t *testing.T) { output := runTestProg(t, "testprogcgo", "GoroutineProfile") want := "OK\n" @@ -856,3 +859,13 @@ func TestStackSwitchCallback(t *testing.T) { t.Errorf("expected %q, got %v", want, got) } } + +func TestCgoToGoCallGoexit(t *testing.T) { + if runtime.GOOS == "plan9" || runtime.GOOS == "windows" { + t.Skipf("no pthreads on %s", runtime.GOOS) + } + output := runTestProg(t, "testprogcgo", "CgoToGoCallGoexit") + if !strings.Contains(output, "runtime.Goexit called in a thread that was not created by the Go runtime") { + t.Fatalf("output should contain %s, got %s", "runtime.Goexit called in a thread that was not created by the Go runtime", output) + } +} diff --git a/src/runtime/crash_test.go b/src/runtime/crash_test.go index 69e1034f..236c32ea 100644 --- a/src/runtime/crash_test.go +++ b/src/runtime/crash_test.go @@ -32,8 +32,11 @@ const entrypointVar = "RUNTIME_TEST_ENTRYPOINT" func TestMain(m *testing.M) { switch entrypoint := os.Getenv(entrypointVar); entrypoint { - case "crash": - crash() + case "panic": + crashViaPanic() + panic("unreachable") + case "trap": + crashViaTrap() panic("unreachable") default: log.Fatalf("invalid %s: %q", entrypointVar, entrypoint) @@ -621,8 +624,11 @@ func TestConcurrentMapWrites(t *testing.T) { } testenv.MustHaveGoRun(t) output := runTestProg(t, "testprog", "concurrentMapWrites") - want := "fatal error: concurrent map writes" - if !strings.HasPrefix(output, want) { + want := "fatal error: concurrent map writes\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { t.Fatalf("output does not start with %q:\n%s", want, output) } } @@ -632,8 +638,11 @@ func TestConcurrentMapReadWrite(t *testing.T) { } testenv.MustHaveGoRun(t) output := runTestProg(t, "testprog", "concurrentMapReadWrite") - want := "fatal error: concurrent map read and map write" - if !strings.HasPrefix(output, want) { + want := "fatal error: concurrent map read and map write\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { t.Fatalf("output does not start with %q:\n%s", want, output) } } @@ -643,12 +652,42 @@ func TestConcurrentMapIterateWrite(t *testing.T) { } testenv.MustHaveGoRun(t) output := runTestProg(t, "testprog", "concurrentMapIterateWrite") - want := "fatal error: concurrent map iteration and map write" - if !strings.HasPrefix(output, want) { + want := "fatal error: concurrent map iteration and map write\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { t.Fatalf("output does not start with %q:\n%s", want, output) } } +func TestConcurrentMapWritesIssue69447(t *testing.T) { + testenv.MustHaveGoRun(t) + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 200; i++ { + output := runBuiltTestProg(t, exe, "concurrentMapWrites") + if output == "" { + // If we didn't detect an error, that's ok. + // This case makes this test not flaky like + // the other ones above. + // (More correctly, this case makes this test flaky + // in the other direction, in that it might not + // detect a problem even if there is one.) + continue + } + want := "fatal error: concurrent map writes\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { + t.Fatalf("output does not start with %q:\n%s", want, output) + } + } +} + type point struct { x, y *int } diff --git a/src/runtime/debug/mod.go b/src/runtime/debug/mod.go index a4705605..53bbf1d8 100644 --- a/src/runtime/debug/mod.go +++ b/src/runtime/debug/mod.go @@ -101,6 +101,7 @@ func quoteValue(value string) bool { return strings.ContainsAny(value, " \t\r\n\"`") } +// String returns a string representation of a [BuildInfo]. func (bi *BuildInfo) String() string { buf := new(strings.Builder) if bi.GoVersion != "" { @@ -146,6 +147,12 @@ func (bi *BuildInfo) String() string { return buf.String() } +// ParseBuildInfo parses the string returned by [*BuildInfo.String], +// restoring the original BuildInfo, +// except that the GoVersion field is not set. +// Programs should normally not call this function, +// but instead call [ReadBuildInfo], [debug/buildinfo.ReadFile], +// or [debug/buildinfo.Read]. func ParseBuildInfo(data string) (bi *BuildInfo, err error) { lineNum := 1 defer func() { @@ -154,7 +161,7 @@ func ParseBuildInfo(data string) (bi *BuildInfo, err error) { } }() - var ( + const ( pathLine = "path\t" modLine = "mod\t" depLine = "dep\t" @@ -195,7 +202,7 @@ func ParseBuildInfo(data string) (bi *BuildInfo, err error) { switch { case strings.HasPrefix(line, pathLine): elem := line[len(pathLine):] - bi.Path = string(elem) + bi.Path = elem case strings.HasPrefix(line, modLine): elem := strings.Split(line[len(modLine):], tab) last = &bi.Main @@ -220,9 +227,9 @@ func ParseBuildInfo(data string) (bi *BuildInfo, err error) { return nil, fmt.Errorf("replacement with no module on previous line") } last.Replace = &Module{ - Path: string(elem[0]), - Version: string(elem[1]), - Sum: string(elem[2]), + Path: elem[0], + Version: elem[1], + Sum: elem[2], } last = nil case strings.HasPrefix(line, buildLine): diff --git a/src/runtime/debug/stack.go b/src/runtime/debug/stack.go index d7a860b7..c4c3be14 100644 --- a/src/runtime/debug/stack.go +++ b/src/runtime/debug/stack.go @@ -52,7 +52,7 @@ func SetCrashOutput(f *os.File, opts CrashOptions) error { // The runtime will write to this file descriptor from // low-level routines during a panic, possibly without // a G, so we must call f.Fd() eagerly. This creates a - // danger that that the file descriptor is no longer + // danger that the file descriptor is no longer // valid at the time of the write, because the caller // (incorrectly) called f.Close() and the kernel // reissued the fd in a later call to open(2), leading diff --git a/src/runtime/debug_test.go b/src/runtime/debug_test.go index 1c00d2fb..37093cd8 100644 --- a/src/runtime/debug_test.go +++ b/src/runtime/debug_test.go @@ -9,13 +9,15 @@ // spends all of its time in the race runtime, which isn't a safe // point. -//go:build (amd64 || arm64 || ppc64le) && linux && !race +//go:build (amd64 || arm64 || loong64 || ppc64le) && linux && !race package runtime_test import ( "fmt" "internal/abi" + "internal/asan" + "internal/msan" "math" "os" "regexp" @@ -32,6 +34,14 @@ func startDebugCallWorker(t *testing.T) (g *runtime.G, after func()) { // a debugger. skipUnderDebugger(t) + // asan/msan instrumentation interferes with tests since we might + // inject debugCallV2 while in the asan/msan runtime. This is a + // problem for doing things like running the GC or taking stack + // traces. Not sure why this is happening yet, but skip for now. + if msan.Enabled || asan.Enabled { + t.Skip("debugCallV2 is injected erroneously during asan/msan runtime calls; skipping") + } + // This can deadlock if there aren't enough threads or if a GC // tries to interrupt an atomic loop (see issue #10958). Execute // an extra GC to ensure even the sweep phase is done (out of diff --git a/src/runtime/debugcall.go b/src/runtime/debugcall.go index fee4116a..e6554475 100644 --- a/src/runtime/debugcall.go +++ b/src/runtime/debugcall.go @@ -5,12 +5,13 @@ // Though the debug call function feature is not enabled on // ppc64, inserted ppc64 to avoid missing Go declaration error // for debugCallPanicked while building runtime.test -//go:build amd64 || arm64 || ppc64le || ppc64 +//go:build amd64 || arm64 || loong64 || ppc64le || ppc64 package runtime import ( "internal/abi" + "internal/runtime/sys" "unsafe" ) @@ -34,7 +35,7 @@ func debugCallCheck(pc uintptr) string { if getg() != getg().m.curg { return debugCallSystemStack } - if sp := getcallersp(); !(getg().stack.lo < sp && sp <= getg().stack.hi) { + if sp := sys.GetCallerSP(); !(getg().stack.lo < sp && sp <= getg().stack.hi) { // Fast syscalls (nanotime) and racecall switch to the // g0 stack without switching g. We can't safely make // a call in this state. (We can't even safely @@ -106,7 +107,7 @@ func debugCallCheck(pc uintptr) string { //go:nosplit func debugCallWrap(dispatch uintptr) { var lockedExt uint32 - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() gp := getg() // Lock ourselves to the OS thread. diff --git a/src/runtime/debuglog.go b/src/runtime/debuglog.go index 695cd204..ad33ef8b 100644 --- a/src/runtime/debuglog.go +++ b/src/runtime/debuglog.go @@ -12,13 +12,23 @@ // // This facility can be enabled by passing -tags debuglog when // building. Without this tag, dlog calls compile to nothing. +// +// Implementation notes +// +// There are two implementations of the dlog interface: dloggerImpl and +// dloggerFake. dloggerFake is a no-op implementation. dlogger is type-aliased +// to one or the other depending on the debuglog build tag. However, both types +// always exist and are always built. This helps ensure we compile as much of +// the implementation as possible in the default build configuration, while also +// enabling us to achieve good test coverage of the real debuglog implementation +// even when the debuglog build tag is not set. package runtime import ( "internal/abi" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -48,11 +58,20 @@ const debugLogStringLimit = debugLogBytes / 8 // //go:nosplit //go:nowritebarrierrec -func dlog() *dlogger { - if !dlogEnabled { - return nil - } +func dlog() dlogger { + // dlog1 is defined to either dlogImpl or dlogFake. + return dlog1() +} +//go:nosplit +//go:nowritebarrierrec +func dlogFake() dloggerFake { + return dloggerFake{} +} + +//go:nosplit +//go:nowritebarrierrec +func dlogImpl() *dloggerImpl { // Get the time. tick, nano := uint64(cputicks()), uint64(nanotime()) @@ -63,7 +82,7 @@ func dlog() *dlogger { // global pool. if l == nil { allp := (*uintptr)(unsafe.Pointer(&allDloggers)) - all := (*dlogger)(unsafe.Pointer(atomic.Loaduintptr(allp))) + all := (*dloggerImpl)(unsafe.Pointer(atomic.Loaduintptr(allp))) for l1 := all; l1 != nil; l1 = l1.allLink { if l1.owned.Load() == 0 && l1.owned.CompareAndSwap(0, 1) { l = l1 @@ -76,7 +95,7 @@ func dlog() *dlogger { if l == nil { // Use sysAllocOS instead of sysAlloc because we want to interfere // with the runtime as little as possible, and sysAlloc updates accounting. - l = (*dlogger)(sysAllocOS(unsafe.Sizeof(dlogger{}))) + l = (*dloggerImpl)(sysAllocOS(unsafe.Sizeof(dloggerImpl{}))) if l == nil { throw("failed to allocate debug log") } @@ -87,7 +106,7 @@ func dlog() *dlogger { headp := (*uintptr)(unsafe.Pointer(&allDloggers)) for { head := atomic.Loaduintptr(headp) - l.allLink = (*dlogger)(unsafe.Pointer(head)) + l.allLink = (*dloggerImpl)(unsafe.Pointer(head)) if atomic.Casuintptr(headp, head, uintptr(unsafe.Pointer(l))) { break } @@ -119,16 +138,16 @@ func dlog() *dlogger { return l } -// A dlogger writes to the debug log. +// A dloggerImpl writes to the debug log. // -// To obtain a dlogger, call dlog(). When done with the dlogger, call +// To obtain a dloggerImpl, call dlog(). When done with the dloggerImpl, call // end(). -type dlogger struct { +type dloggerImpl struct { _ sys.NotInHeap w debugLogWriter // allLink is the next dlogger in the allDloggers list. - allLink *dlogger + allLink *dloggerImpl // owned indicates that this dlogger is owned by an M. This is // accessed atomically. @@ -138,14 +157,16 @@ type dlogger struct { // allDloggers is a list of all dloggers, linked through // dlogger.allLink. This is accessed atomically. This is prepend only, // so it doesn't need to protect against ABA races. -var allDloggers *dlogger +var allDloggers *dloggerImpl + +// A dloggerFake is a no-op implementation of dlogger. +type dloggerFake struct{} //go:nosplit -func (l *dlogger) end() { - if !dlogEnabled { - return - } +func (l dloggerFake) end() {} +//go:nosplit +func (l *dloggerImpl) end() { // Fill in framing header. size := l.w.write - l.w.r.end if !l.w.writeFrameAt(l.w.r.end, size) { @@ -181,10 +202,10 @@ const ( ) //go:nosplit -func (l *dlogger) b(x bool) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) b(x bool) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) b(x bool) *dloggerImpl { if x { l.w.byte(debugLogBoolTrue) } else { @@ -194,85 +215,112 @@ func (l *dlogger) b(x bool) *dlogger { } //go:nosplit -func (l *dlogger) i(x int) *dlogger { +func (l dloggerFake) i(x int) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i(x int) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i8(x int8) *dlogger { +func (l dloggerFake) i8(x int8) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i8(x int8) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i16(x int16) *dlogger { +func (l dloggerFake) i16(x int16) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i16(x int16) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i32(x int32) *dlogger { +func (l dloggerFake) i32(x int32) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i32(x int32) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i64(x int64) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) i64(x int64) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i64(x int64) *dloggerImpl { l.w.byte(debugLogInt) l.w.varint(x) return l } //go:nosplit -func (l *dlogger) u(x uint) *dlogger { +func (l dloggerFake) u(x uint) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u(x uint) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) uptr(x uintptr) *dlogger { +func (l dloggerFake) uptr(x uintptr) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) uptr(x uintptr) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u8(x uint8) *dlogger { +func (l dloggerFake) u8(x uint8) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u8(x uint8) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u16(x uint16) *dlogger { +func (l dloggerFake) u16(x uint16) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u16(x uint16) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u32(x uint32) *dlogger { +func (l dloggerFake) u32(x uint32) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u32(x uint32) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u64(x uint64) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) u64(x uint64) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u64(x uint64) *dloggerImpl { l.w.byte(debugLogUint) l.w.uvarint(x) return l } //go:nosplit -func (l *dlogger) hex(x uint64) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) hex(x uint64) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) hex(x uint64) *dloggerImpl { l.w.byte(debugLogHex) l.w.uvarint(x) return l } //go:nosplit -func (l *dlogger) p(x any) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) p(x any) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) p(x any) *dloggerImpl { l.w.byte(debugLogPtr) if x == nil { l.w.uvarint(0) @@ -289,11 +337,10 @@ func (l *dlogger) p(x any) *dlogger { } //go:nosplit -func (l *dlogger) s(x string) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) s(x string) dloggerFake { return l } +//go:nosplit +func (l *dloggerImpl) s(x string) *dloggerImpl { strData := unsafe.StringData(x) datap := &firstmoduledata if len(x) > 4 && datap.etext <= uintptr(unsafe.Pointer(strData)) && uintptr(unsafe.Pointer(strData)) < datap.end { @@ -325,20 +372,20 @@ func (l *dlogger) s(x string) *dlogger { } //go:nosplit -func (l *dlogger) pc(x uintptr) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) pc(x uintptr) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) pc(x uintptr) *dloggerImpl { l.w.byte(debugLogPC) l.w.uvarint(uint64(x)) return l } //go:nosplit -func (l *dlogger) traceback(x []uintptr) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) traceback(x []uintptr) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) traceback(x []uintptr) *dloggerImpl { l.w.byte(debugLogTraceback) l.w.uvarint(uint64(len(x))) for _, pc := range x { @@ -693,10 +740,12 @@ func (r *debugLogReader) printVal() bool { // printDebugLog prints the debug log. func printDebugLog() { - if !dlogEnabled { - return + if dlogEnabled { + printDebugLogImpl() } +} +func printDebugLogImpl() { // This function should not panic or throw since it is used in // the fatal panic path and this may deadlock. @@ -704,7 +753,7 @@ func printDebugLog() { // Get the list of all debug logs. allp := (*uintptr)(unsafe.Pointer(&allDloggers)) - all := (*dlogger)(unsafe.Pointer(atomic.Loaduintptr(allp))) + all := (*dloggerImpl)(unsafe.Pointer(atomic.Loaduintptr(allp))) // Count the logs. n := 0 diff --git a/src/runtime/debuglog_off.go b/src/runtime/debuglog_off.go index fa3be39c..4eb59fa6 100644 --- a/src/runtime/debuglog_off.go +++ b/src/runtime/debuglog_off.go @@ -8,12 +8,18 @@ package runtime const dlogEnabled = false +type dlogger = dloggerFake + +func dlog1() dloggerFake { + return dlogFake() +} + type dlogPerM struct{} -func getCachedDlogger() *dlogger { +func getCachedDlogger() *dloggerImpl { return nil } -func putCachedDlogger(l *dlogger) bool { +func putCachedDlogger(l *dloggerImpl) bool { return false } diff --git a/src/runtime/debuglog_on.go b/src/runtime/debuglog_on.go index b8150202..99773129 100644 --- a/src/runtime/debuglog_on.go +++ b/src/runtime/debuglog_on.go @@ -8,21 +8,32 @@ package runtime const dlogEnabled = true +// dlogger is the underlying implementation of the dlogger interface, selected +// at build time. +// +// We use a type alias instead of struct embedding so that the dlogger type is +// identical to the type returned by method chaining on the methods of this type. +type dlogger = *dloggerImpl + +func dlog1() *dloggerImpl { + return dlogImpl() +} + // dlogPerM is the per-M debug log data. This is embedded in the m // struct. type dlogPerM struct { - dlogCache *dlogger + dlogCache *dloggerImpl } // getCachedDlogger returns a cached dlogger if it can do so // efficiently, or nil otherwise. The returned dlogger will be owned. -func getCachedDlogger() *dlogger { +func getCachedDlogger() *dloggerImpl { mp := acquirem() // We don't return a cached dlogger if we're running on the // signal stack in case the signal arrived while in // get/putCachedDlogger. (Too bad we don't have non-atomic // exchange!) - var l *dlogger + var l *dloggerImpl if getg() != mp.gsignal { l = mp.dlogCache mp.dlogCache = nil @@ -33,7 +44,7 @@ func getCachedDlogger() *dlogger { // putCachedDlogger attempts to return l to the local cache. It // returns false if this fails. -func putCachedDlogger(l *dlogger) bool { +func putCachedDlogger(l *dloggerImpl) bool { mp := acquirem() if getg() != mp.gsignal && mp.dlogCache == nil { mp.dlogCache = l diff --git a/src/runtime/debuglog_test.go b/src/runtime/debuglog_test.go index 18c54a81..d958c037 100644 --- a/src/runtime/debuglog_test.go +++ b/src/runtime/debuglog_test.go @@ -24,18 +24,16 @@ package runtime_test import ( "fmt" - "internal/testenv" "regexp" "runtime" "strings" "sync" - "sync/atomic" "testing" ) func skipDebugLog(t *testing.T) { - if !runtime.DlogEnabled { - t.Skip("debug log disabled (rebuild with -tags debuglog)") + if runtime.DlogEnabled { + t.Skip("debug log tests disabled to avoid collisions with real debug logs") } } @@ -83,28 +81,63 @@ func TestDebugLogSym(t *testing.T) { func TestDebugLogInterleaving(t *testing.T) { skipDebugLog(t) runtime.ResetDebugLog() - var wg sync.WaitGroup - done := int32(0) - wg.Add(1) - go func() { - // Encourage main goroutine to move around to - // different Ms and Ps. - for atomic.LoadInt32(&done) == 0 { - runtime.Gosched() - } - wg.Done() - }() - var want strings.Builder - for i := 0; i < 1000; i++ { - runtime.Dlog().I(i).End() - fmt.Fprintf(&want, "[] %d\n", i) - runtime.Gosched() - } - atomic.StoreInt32(&done, 1) - wg.Wait() + n1 := runtime.CountDebugLog() + t.Logf("number of log shards at start: %d", n1) + + const limit = 1000 + const concurrency = 10 + + // Start several goroutines writing to the log simultaneously. + var wg sync.WaitGroup + i := 0 + chans := make([]chan bool, concurrency) + for gid := range concurrency { + chans[gid] = make(chan bool) + wg.Add(1) + go func() { + defer wg.Done() + var log *runtime.Dlogger + for { + <-chans[gid] + if log != nil { + log.End() + } + next := chans[(gid+1)%len(chans)] + if i >= limit { + close(next) + break + } + // Log an entry, but *don't* release the log shard until its our + // turn again. This should result in at least n=concurrency log + // shards. + log = runtime.Dlog().I(i) + i++ + // Wake up the next logger goroutine. + next <- true + } + }() + } + // Start the chain reaction. + chans[0] <- true + + // Wait for them to finish and get the log. + wg.Wait() gotFull := runtime.DumpDebugLog() got := dlogCanonicalize(gotFull) + + n2 := runtime.CountDebugLog() + t.Logf("number of log shards at end: %d", n2) + if n2 < concurrency { + t.Errorf("created %d log shards, expected >= %d", n2, concurrency) + } + + // Construct the desired output. + var want strings.Builder + for i := 0; i < limit; i++ { + fmt.Fprintf(&want, "[] %d\n", i) + } + if got != want.String() { // Since the timestamps are useful in understand // failures of this test, we print the uncanonicalized @@ -156,14 +189,3 @@ func TestDebugLogLongString(t *testing.T) { t.Fatalf("want %q, got %q", want, got) } } - -// TestDebugLogBuild verifies that the runtime builds with -tags=debuglog. -func TestDebugLogBuild(t *testing.T) { - testenv.MustHaveGoBuild(t) - - // It doesn't matter which program we build, anything will rebuild the - // runtime. - if _, err := buildTestProg(t, "testprog", "-tags=debuglog"); err != nil { - t.Fatal(err) - } -} diff --git a/src/runtime/defer_test.go b/src/runtime/defer_test.go index d73202ae..e3d0d077 100644 --- a/src/runtime/defer_test.go +++ b/src/runtime/defer_test.go @@ -5,8 +5,8 @@ package runtime_test import ( - "reflect" "runtime" + "slices" "testing" ) @@ -83,7 +83,7 @@ func TestConditionalDefers(t *testing.T) { t.Fatal("expected panic") } want := []int{4, 2, 1} - if !reflect.DeepEqual(want, list) { + if !slices.Equal(want, list) { t.Fatalf("wanted %v, got %v", want, list) } diff --git a/src/runtime/defs_linux_loong64.go b/src/runtime/defs_linux_loong64.go index 692d8c78..b9837251 100644 --- a/src/runtime/defs_linux_loong64.go +++ b/src/runtime/defs_linux_loong64.go @@ -184,6 +184,7 @@ type sigcontext struct { sc_pc uint64 sc_regs [32]uint64 sc_flags uint32 + sc_pad0 [1]uint32 sc_extcontext [0]uint64 } diff --git a/src/runtime/error.go b/src/runtime/error.go index 406f36ca..9017c043 100644 --- a/src/runtime/error.go +++ b/src/runtime/error.go @@ -7,6 +7,7 @@ package runtime import ( "internal/abi" "internal/bytealg" + "internal/runtime/sys" ) // The Error interface identifies a run time error. @@ -329,7 +330,7 @@ func printindented(s string) { // // It is called from the generated wrapper code. func panicwrap() { - pc := getcallerpc() + pc := sys.GetCallerPC() name := funcNameForPrint(funcname(findfunc(pc))) // name is something like "main.(*T).F". // We want to extract pkg ("main"), typ ("T"), and meth ("F"). diff --git a/src/runtime/example_test.go b/src/runtime/example_test.go index dcb8f779..eae9dbd7 100644 --- a/src/runtime/example_test.go +++ b/src/runtime/example_test.go @@ -32,15 +32,14 @@ func ExampleFrames() { for { frame, more := frames.Next() - // Process this frame. - // - // To keep this example's output stable - // even if there are changes in the testing package, - // stop unwinding when we leave package runtime. - if !strings.Contains(frame.File, "runtime/") { + // Canonicalize function name and skip callers of this function + // for predictable example output. + // You probably don't need this in your own code. + function := strings.ReplaceAll(frame.Function, "main.main", "runtime_test.ExampleFrames") + fmt.Printf("- more:%v | %s\n", more, function) + if function == "runtime_test.ExampleFrames" { break } - fmt.Printf("- more:%v | %s\n", more, frame.Function) // Check whether there are more frames to process after this one. if !more { diff --git a/src/runtime/export_debug_loong64_test.go b/src/runtime/export_debug_loong64_test.go new file mode 100644 index 00000000..eaaf3598 --- /dev/null +++ b/src/runtime/export_debug_loong64_test.go @@ -0,0 +1,227 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && linux + +package runtime + +import ( + "internal/abi" + "internal/goarch" + "unsafe" +) + +type sigContext struct { + savedRegs sigcontext +} + +func sigctxtSetContextRegister(ctxt *sigctxt, x uint64) { + ctxt.regs().sc_regs[29] = x +} + +func sigctxtAtTrapInstruction(ctxt *sigctxt) bool { + return *(*uint32)(unsafe.Pointer(ctxt.sigpc())) == 0x002a0000 // BREAK 0 +} + +func sigctxtStatus(ctxt *sigctxt) uint64 { + return ctxt.r19() +} + +func (h *debugCallHandler) saveSigContext(ctxt *sigctxt) { + sp := ctxt.sp() + sp -= goarch.PtrSize + ctxt.set_sp(sp) + *(*uint64)(unsafe.Pointer(uintptr(sp))) = ctxt.link() // save the current lr + ctxt.set_link(ctxt.pc()) // set new lr to the current pc + // Write the argument frame size. + *(*uintptr)(unsafe.Pointer(uintptr(sp - 8))) = h.argSize + // Save current registers. + h.sigCtxt.savedRegs = *ctxt.regs() +} + +// case 0 +func (h *debugCallHandler) debugCallRun(ctxt *sigctxt) { + sp := ctxt.sp() + memmove(unsafe.Pointer(uintptr(sp)+8), h.argp, h.argSize) + if h.regArgs != nil { + storeRegArgs(ctxt.regs(), h.regArgs) + } + // Push return PC, which should be the signal PC+4, because + // the signal PC is the PC of the trap instruction itself. + ctxt.set_link(ctxt.pc() + 4) + // Set PC to call and context register. + ctxt.set_pc(uint64(h.fv.fn)) + sigctxtSetContextRegister(ctxt, uint64(uintptr(unsafe.Pointer(h.fv)))) +} + +// case 1 +func (h *debugCallHandler) debugCallReturn(ctxt *sigctxt) { + sp := ctxt.sp() + memmove(h.argp, unsafe.Pointer(uintptr(sp)+8), h.argSize) + if h.regArgs != nil { + loadRegArgs(h.regArgs, ctxt.regs()) + } + // Restore the old lr from *sp + olr := *(*uint64)(unsafe.Pointer(uintptr(sp))) + ctxt.set_link(olr) + pc := ctxt.pc() + ctxt.set_pc(pc + 4) // step to next instruction +} + +// case 2 +func (h *debugCallHandler) debugCallPanicOut(ctxt *sigctxt) { + sp := ctxt.sp() + memmove(unsafe.Pointer(&h.panic), unsafe.Pointer(uintptr(sp)+8), 2*goarch.PtrSize) + ctxt.set_pc(ctxt.pc() + 4) +} + +// case 8 +func (h *debugCallHandler) debugCallUnsafe(ctxt *sigctxt) { + sp := ctxt.sp() + reason := *(*string)(unsafe.Pointer(uintptr(sp) + 8)) + h.err = plainError(reason) + ctxt.set_pc(ctxt.pc() + 4) +} + +// case 16 +func (h *debugCallHandler) restoreSigContext(ctxt *sigctxt) { + // Restore all registers except for pc and sp + pc, sp := ctxt.pc(), ctxt.sp() + *ctxt.regs() = h.sigCtxt.savedRegs + ctxt.set_pc(pc + 4) + ctxt.set_sp(sp) +} + +func getVal32(base uintptr, off uintptr) uint32 { + return *(*uint32)(unsafe.Pointer(base + off)) +} + +func getVal64(base uintptr, off uintptr) uint64 { + return *(*uint64)(unsafe.Pointer(base + off)) +} + +func setVal64(base uintptr, off uintptr, val uint64) { + *(*uint64)(unsafe.Pointer(base + off)) = val +} + +// Layout for sigcontext on linux/loong64: arch/loongarch/include/uapi/asm/sigcontext.h +// +// sc_extcontext | sctx_info +// ------------------------------------------ +// | {fpu,lsx,lasx}_context +// --------------------------- +// | sctx_info +// --------------------------- +// | lbt_context +// + +const ( + INVALID_MAGIC uint32 = 0 + FPU_CTX_MAGIC = 0x46505501 + LSX_CTX_MAGIC = 0x53580001 + LASX_CTX_MAGIC = 0x41535801 + LBT_CTX_MAGIC = 0x42540001 +) + +const ( + SCTX_INFO_SIZE = 4 + 4 + 8 + FPU_CTX_SIZE = 8*32 + 8 + 4 // fpu context size + LSX_CTX_SIZE = 8*64 + 8 + 4 // lsx context size + LASX_CTX_SIZE = 8*128 + 8 + 4 // lasx context size + LBT_CTX_SIZE = 8*4 + 4 + 4 // lbt context size +) + +// storeRegArgs sets up argument registers in the signal context state +// from an abi.RegArgs. +// +// Both src and dst must be non-nil. +func storeRegArgs(dst *sigcontext, src *abi.RegArgs) { + // R4..R19 are used to pass int arguments in registers on loong64 + for i := 0; i < abi.IntArgRegs; i++ { + dst.sc_regs[i+4] = (uint64)(src.Ints[i]) + } + + // F0..F15 are used to pass float arguments in registers on loong64 + offset := (uintptr)(0) + baseAddr := (uintptr)(unsafe.Pointer(&dst.sc_extcontext)) + + for { + magic := getVal32(baseAddr, offset) + size := getVal32(baseAddr, offset+4) + + switch magic { + case INVALID_MAGIC: + return + + case FPU_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + setVal64(baseAddr, ((uintptr)(i*8) + offset), src.Floats[i]) + } + return + + case LSX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + setVal64(baseAddr, ((uintptr)(i*16) + offset), src.Floats[i]) + } + return + + case LASX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + setVal64(baseAddr, ((uintptr)(i*32) + offset), src.Floats[i]) + } + return + + case LBT_CTX_MAGIC: + offset += uintptr(size) + } + } +} + +func loadRegArgs(dst *abi.RegArgs, src *sigcontext) { + // R4..R19 are used to pass int arguments in registers on loong64 + for i := 0; i < abi.IntArgRegs; i++ { + dst.Ints[i] = uintptr(src.sc_regs[i+4]) + } + + // F0..F15 are used to pass float arguments in registers on loong64 + offset := (uintptr)(0) + baseAddr := (uintptr)(unsafe.Pointer(&src.sc_extcontext)) + + for { + magic := getVal32(baseAddr, offset) + size := getVal32(baseAddr, (offset + 4)) + + switch magic { + case INVALID_MAGIC: + return + + case FPU_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + dst.Floats[i] = getVal64(baseAddr, (uintptr(i*8) + offset)) + } + return + + case LSX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + dst.Floats[i] = getVal64(baseAddr, (uintptr(i*16) + offset)) + } + return + + case LASX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + dst.Floats[i] = getVal64(baseAddr, (uintptr(i*32) + offset)) + } + return + + case LBT_CTX_MAGIC: + offset += uintptr(size) + } + } +} diff --git a/src/runtime/export_debug_test.go b/src/runtime/export_debug_test.go index 4e0a4ef9..96f6fd9e 100644 --- a/src/runtime/export_debug_test.go +++ b/src/runtime/export_debug_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (amd64 || arm64 || ppc64le) && linux +//go:build (amd64 || arm64 || loong64 || ppc64le) && linux package runtime diff --git a/src/runtime/export_debuglog_test.go b/src/runtime/export_debuglog_test.go index 04ac79f3..c370a793 100644 --- a/src/runtime/export_debuglog_test.go +++ b/src/runtime/export_debuglog_test.go @@ -12,22 +12,26 @@ const DebugLogBytes = debugLogBytes const DebugLogStringLimit = debugLogStringLimit -var Dlog = dlog +type Dlogger = dloggerImpl -func (l *dlogger) End() { l.end() } -func (l *dlogger) B(x bool) *dlogger { return l.b(x) } -func (l *dlogger) I(x int) *dlogger { return l.i(x) } -func (l *dlogger) I16(x int16) *dlogger { return l.i16(x) } -func (l *dlogger) U64(x uint64) *dlogger { return l.u64(x) } -func (l *dlogger) Hex(x uint64) *dlogger { return l.hex(x) } -func (l *dlogger) P(x any) *dlogger { return l.p(x) } -func (l *dlogger) S(x string) *dlogger { return l.s(x) } -func (l *dlogger) PC(x uintptr) *dlogger { return l.pc(x) } +func Dlog() *Dlogger { + return dlogImpl() +} + +func (l *dloggerImpl) End() { l.end() } +func (l *dloggerImpl) B(x bool) *dloggerImpl { return l.b(x) } +func (l *dloggerImpl) I(x int) *dloggerImpl { return l.i(x) } +func (l *dloggerImpl) I16(x int16) *dloggerImpl { return l.i16(x) } +func (l *dloggerImpl) U64(x uint64) *dloggerImpl { return l.u64(x) } +func (l *dloggerImpl) Hex(x uint64) *dloggerImpl { return l.hex(x) } +func (l *dloggerImpl) P(x any) *dloggerImpl { return l.p(x) } +func (l *dloggerImpl) S(x string) *dloggerImpl { return l.s(x) } +func (l *dloggerImpl) PC(x uintptr) *dloggerImpl { return l.pc(x) } func DumpDebugLog() string { gp := getg() gp.writebuf = make([]byte, 0, 1<<20) - printDebugLog() + printDebugLogImpl() buf := gp.writebuf gp.writebuf = nil @@ -44,3 +48,13 @@ func ResetDebugLog() { } startTheWorld(stw) } + +func CountDebugLog() int { + stw := stopTheWorld(stwForTestResetDebugLog) + i := 0 + for l := allDloggers; l != nil; l = l.allLink { + i++ + } + startTheWorld(stw) + return i +} diff --git a/src/runtime/export_map_noswiss_test.go b/src/runtime/export_map_noswiss_test.go new file mode 100644 index 00000000..4638afa6 --- /dev/null +++ b/src/runtime/export_map_noswiss_test.go @@ -0,0 +1,64 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +package runtime + +import ( + "internal/abi" + "unsafe" +) + +const RuntimeHmapSize = unsafe.Sizeof(hmap{}) + +func OverLoadFactor(count int, B uint8) bool { + return overLoadFactor(count, B) +} + +func MapBucketsCount(m map[int]int) int { + h := *(**hmap)(unsafe.Pointer(&m)) + return 1 << h.B +} + +func MapBucketsPointerIsNil(m map[int]int) bool { + h := *(**hmap)(unsafe.Pointer(&m)) + return h.buckets == nil +} + +func MapTombstoneCheck(m map[int]int) { + // Make sure emptyOne and emptyRest are distributed correctly. + // We should have a series of filled and emptyOne cells, followed by + // a series of emptyRest cells. + h := *(**hmap)(unsafe.Pointer(&m)) + i := any(m) + t := *(**maptype)(unsafe.Pointer(&i)) + + for x := 0; x < 1<= n && b.tophash[i] != emptyRest { + panic("late non-emptyRest") + } + if k == n-1 && b.tophash[i] == emptyOne { + panic("last non-emptyRest entry is emptyOne") + } + k++ + } + } + } +} diff --git a/src/runtime/export_map_swiss_test.go b/src/runtime/export_map_swiss_test.go new file mode 100644 index 00000000..55a7d6ff --- /dev/null +++ b/src/runtime/export_map_swiss_test.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package runtime + +func MapTombstoneCheck(m map[int]int) { + // TODO +} diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 4502fa72..79d83b3a 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -11,7 +11,7 @@ import ( "internal/goarch" "internal/goos" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -94,9 +94,9 @@ func Netpoll(delta int64) { }) } -func GCMask(x any) (ret []byte) { +func PointerMask(x any) (ret []byte) { systemstack(func() { - ret = getgcmask(x) + ret = pointerMask(x) }) return } @@ -481,22 +481,6 @@ func (rw *RWMutex) Unlock() { rw.rw.unlock() } -const RuntimeHmapSize = unsafe.Sizeof(hmap{}) - -func MapBucketsCount(m map[int]int) int { - h := *(**hmap)(unsafe.Pointer(&m)) - return 1 << h.B -} - -func MapBucketsPointerIsNil(m map[int]int) bool { - h := *(**hmap)(unsafe.Pointer(&m)) - return h.buckets == nil -} - -func OverLoadFactor(count int, B uint8) bool { - return overLoadFactor(count, B) -} - func LockOSCounts() (external, internal uint32) { gp := getg() if gp.m.lockedExt+gp.m.lockedInt == 0 { @@ -514,7 +498,7 @@ func LockOSCounts() (external, internal uint32) { //go:noinline func TracebackSystemstack(stk []uintptr, i int) int { if i == 0 { - pc, sp := getcallerpc(), getcallersp() + pc, sp := sys.GetCallerPC(), sys.GetCallerSP() var u unwinder u.initAt(pc, sp, 0, getg(), unwindJumpStack) // Don't ignore errors, for testing return tracebackPCs(&u, 0, stk) @@ -597,7 +581,7 @@ func unexportedPanicForTesting(b []byte, i int) byte { func G0StackOverflow() { systemstack(func() { g0 := getg() - sp := getcallersp() + sp := sys.GetCallerSP() // The stack bounds for g0 stack is not always precise. // Use an artificially small stack, to trigger a stack overflow // without actually run out of the system stack (which may seg fault). @@ -614,42 +598,6 @@ func stackOverflow(x *byte) { stackOverflow(&buf[0]) } -func MapTombstoneCheck(m map[int]int) { - // Make sure emptyOne and emptyRest are distributed correctly. - // We should have a series of filled and emptyOne cells, followed by - // a series of emptyRest cells. - h := *(**hmap)(unsafe.Pointer(&m)) - i := any(m) - t := *(**maptype)(unsafe.Pointer(&i)) - - for x := 0; x < 1<= n && b.tophash[i] != emptyRest { - panic("late non-emptyRest") - } - if k == n-1 && b.tophash[i] == emptyOne { - panic("last non-emptyRest entry is emptyOne") - } - k++ - } - } - } -} - func RunGetgThreadSwitchTest() { // Test that getg works correctly with thread switch. // With gccgo, if we generate getg inlined, the backend @@ -1223,6 +1171,9 @@ func PageCachePagesLeaked() (leaked uintptr) { return } +var ProcYield = procyield +var OSYield = osyield + type Mutex = mutex var Lock = lock @@ -1313,7 +1264,7 @@ const ( type TimeHistogram timeHistogram -// Counts returns the counts for the given bucket, subBucket indices. +// Count returns the counts for the given bucket, subBucket indices. // Returns true if the bucket was valid, otherwise returns the counts // for the overflow bucket if bucket > 0 or the underflow bucket if // bucket < 0, and false. @@ -1913,3 +1864,18 @@ func GCMarkDoneResetRestartFlag() { gcDebugMarkDone.restartedDueTo27993 = false releasem(mp) } + +type BitCursor struct { + b bitCursor +} + +func NewBitCursor(buf *byte) BitCursor { + return BitCursor{b: bitCursor{ptr: buf, n: 0}} +} + +func (b BitCursor) Write(data *byte, cnt uintptr) { + b.b.write(data, cnt) +} +func (b BitCursor) Offset(cnt uintptr) BitCursor { + return BitCursor{b: b.b.offset(cnt)} +} diff --git a/src/runtime/export_windows_test.go b/src/runtime/export_windows_test.go index 8bfff0bc..5ff52292 100644 --- a/src/runtime/export_windows_test.go +++ b/src/runtime/export_windows_test.go @@ -6,7 +6,10 @@ package runtime -import "unsafe" +import ( + "internal/runtime/sys" + "unsafe" +) const MaxArgs = maxArgs @@ -31,8 +34,8 @@ func (c ContextStub) GetPC() uintptr { func NewContextStub() *ContextStub { var ctx context - ctx.set_ip(getcallerpc()) - ctx.set_sp(getcallersp()) + ctx.set_ip(sys.GetCallerPC()) + ctx.set_sp(sys.GetCallerSP()) ctx.set_fp(getcallerfp()) return &ContextStub{ctx} } diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 2019be4d..fad19b94 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -294,10 +294,11 @@ import ( // Caller reports file and line number information about function invocations on // the calling goroutine's stack. The argument skip is the number of stack frames -// to ascend, with 0 identifying the caller of Caller. (For historical reasons the -// meaning of skip differs between Caller and [Callers].) The return values report the -// program counter, file name, and line number within the file of the corresponding -// call. The boolean ok is false if it was not possible to recover the information. +// to ascend, with 0 identifying the caller of Caller. (For historical reasons the +// meaning of skip differs between Caller and [Callers].) The return values report +// the program counter, the file name (using forward slashes as path separator, even +// on Windows), and the line number within the file of the corresponding call. +// The boolean ok is false if it was not possible to recover the information. func Caller(skip int) (pc uintptr, file string, line int, ok bool) { rpc := make([]uintptr, 1) n := callers(skip+1, rpc) @@ -336,6 +337,11 @@ var defaultGOROOT string // set by cmd/link // GOROOT returns the root of the Go tree. It uses the // GOROOT environment variable, if set at process start, // or else the root used during the Go build. +// +// Deprecated: The root used during the Go build will not be +// meaningful if the binary is copied to another machine. +// Use the system path to locate the “go” binary, and use +// “go env GOROOT” to find its GOROOT. func GOROOT() string { s := gogetenv("GOROOT") if s != "" { diff --git a/src/runtime/gc_test.go b/src/runtime/gc_test.go index 4b92b200..00280ed1 100644 --- a/src/runtime/gc_test.go +++ b/src/runtime/gc_test.go @@ -6,8 +6,8 @@ package runtime_test import ( "fmt" + "internal/asan" "internal/testenv" - "internal/weak" "math/bits" "math/rand" "os" @@ -21,6 +21,7 @@ import ( "testing" "time" "unsafe" + "weak" ) func TestGcSys(t *testing.T) { @@ -210,6 +211,9 @@ func TestGcZombieReporting(t *testing.T) { } func TestGCTestMoveStackOnNextCall(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan causes this to fail; see #70079") + } t.Parallel() var onStack int // GCTestMoveStackOnNextCall can fail in rare cases if there's @@ -300,6 +304,9 @@ var pointerClassBSS *int var pointerClassData = 42 func TestGCTestPointerClass(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations cause this test to fail; see #70079") + } t.Parallel() check := func(p unsafe.Pointer, want string) { t.Helper() @@ -736,7 +743,7 @@ func BenchmarkMSpanCountAlloc(b *testing.B) { // always rounded up 8 bytes. for _, n := range []int{8, 16, 32, 64, 128} { b.Run(fmt.Sprintf("bits=%d", n*8), func(b *testing.B) { - // Initialize a new byte slice with pseduo-random data. + // Initialize a new byte slice with pseudo-random data. bits := make([]byte, n) rand.Read(bits) @@ -819,7 +826,7 @@ func TestWeakToStrongMarkTermination(t *testing.T) { // Start a GC, and wait a little bit to get something spinning in mark termination. // Simultaneously, fire off another goroutine to disable spinning. If everything's - // working correctly, then weak.Strong will block, so we need to make sure something + // working correctly, then weak.Value will block, so we need to make sure something // prevents the GC from continuing to spin. done := make(chan struct{}) go func() { @@ -827,7 +834,11 @@ func TestWeakToStrongMarkTermination(t *testing.T) { done <- struct{}{} }() go func() { - time.Sleep(100 * time.Millisecond) + // Usleep here instead of time.Sleep. time.Sleep + // can allocate, and if we get unlucky, then it + // can end up stuck in gcMarkDone with nothing to + // wake it. + runtime.Usleep(100000) // 100ms // Let mark termination continue. runtime.SetSpinInGCMarkDone(false) @@ -840,7 +851,7 @@ func TestWeakToStrongMarkTermination(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - wp.Strong() + wp.Value() }() } diff --git a/src/runtime/gcinfo_test.go b/src/runtime/gcinfo_test.go index 5f72caf0..02457f68 100644 --- a/src/runtime/gcinfo_test.go +++ b/src/runtime/gcinfo_test.go @@ -90,7 +90,7 @@ func TestGCInfo(t *testing.T) { } func verifyGCInfo(t *testing.T, name string, p any, mask0 []byte) { - mask := runtime.GCMask(p) + mask := runtime.PointerMask(p) if bytes.HasPrefix(mask, mask0) { // Just the prefix matching is OK. // diff --git a/src/runtime/hash64.go b/src/runtime/hash64.go index 05cdb811..124bb7d7 100644 --- a/src/runtime/hash64.go +++ b/src/runtime/hash64.go @@ -10,7 +10,7 @@ package runtime import ( - "runtime/internal/math" + "internal/runtime/math" "unsafe" ) diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index 95fb62dc..8f2ae34f 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -205,7 +205,7 @@ func dumptype(t *_type) { dwritebyte('.') dwrite(unsafe.Pointer(unsafe.StringData(name)), uintptr(len(name))) } - dumpbool(t.Kind_&abi.KindDirectIface == 0 || t.PtrBytes != 0) + dumpbool(t.Kind_&abi.KindDirectIface == 0 || t.Pointers()) } // dump an object. diff --git a/src/runtime/histogram.go b/src/runtime/histogram.go index 95230d1f..920a9561 100644 --- a/src/runtime/histogram.go +++ b/src/runtime/histogram.go @@ -6,7 +6,7 @@ package runtime import ( "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/iface.go b/src/runtime/iface.go index 41a10ae0..99866864 100644 --- a/src/runtime/iface.go +++ b/src/runtime/iface.go @@ -8,7 +8,7 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -333,7 +333,7 @@ var ( // be used as the second word of an interface value. func convT(t *_type, v unsafe.Pointer) unsafe.Pointer { if raceenabled { - raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convT)) + raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convT)) } if msanenabled { msanread(v, t.Size_) @@ -348,7 +348,7 @@ func convT(t *_type, v unsafe.Pointer) unsafe.Pointer { func convTnoptr(t *_type, v unsafe.Pointer) unsafe.Pointer { // TODO: maybe take size instead of type? if raceenabled { - raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convTnoptr)) + raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convTnoptr)) } if msanenabled { msanread(v, t.Size_) @@ -692,39 +692,16 @@ func iterate_itabs(fn func(*itab)) { } // staticuint64s is used to avoid allocating in convTx for small integer values. -var staticuint64s = [...]uint64{ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +// staticuint64s[0] == 0, staticuint64s[1] == 1, and so forth. +// It is defined in assembler code so that it is read-only. +var staticuint64s [256]uint64 + +// getStaticuint64s is called by the reflect package to get a pointer +// to the read-only array. +// +//go:linkname getStaticuint64s +func getStaticuint64s() *[256]uint64 { + return &staticuint64s } // The linker redirects a reference of a method that it determined diff --git a/src/runtime/ints.s b/src/runtime/ints.s new file mode 100644 index 00000000..b816a2fb --- /dev/null +++ b/src/runtime/ints.s @@ -0,0 +1,264 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +DATA ·staticuint64s+0x000(SB)/8, $0 +DATA ·staticuint64s+0x008(SB)/8, $1 +DATA ·staticuint64s+0x010(SB)/8, $2 +DATA ·staticuint64s+0x018(SB)/8, $3 +DATA ·staticuint64s+0x020(SB)/8, $4 +DATA ·staticuint64s+0x028(SB)/8, $5 +DATA ·staticuint64s+0x030(SB)/8, $6 +DATA ·staticuint64s+0x038(SB)/8, $7 +DATA ·staticuint64s+0x040(SB)/8, $8 +DATA ·staticuint64s+0x048(SB)/8, $9 +DATA ·staticuint64s+0x050(SB)/8, $10 +DATA ·staticuint64s+0x058(SB)/8, $11 +DATA ·staticuint64s+0x060(SB)/8, $12 +DATA ·staticuint64s+0x068(SB)/8, $13 +DATA ·staticuint64s+0x070(SB)/8, $14 +DATA ·staticuint64s+0x078(SB)/8, $15 +DATA ·staticuint64s+0x080(SB)/8, $16 +DATA ·staticuint64s+0x088(SB)/8, $17 +DATA ·staticuint64s+0x090(SB)/8, $18 +DATA ·staticuint64s+0x098(SB)/8, $19 +DATA ·staticuint64s+0x0a0(SB)/8, $20 +DATA ·staticuint64s+0x0a8(SB)/8, $21 +DATA ·staticuint64s+0x0b0(SB)/8, $22 +DATA ·staticuint64s+0x0b8(SB)/8, $23 +DATA ·staticuint64s+0x0c0(SB)/8, $24 +DATA ·staticuint64s+0x0c8(SB)/8, $25 +DATA ·staticuint64s+0x0d0(SB)/8, $26 +DATA ·staticuint64s+0x0d8(SB)/8, $27 +DATA ·staticuint64s+0x0e0(SB)/8, $28 +DATA ·staticuint64s+0x0e8(SB)/8, $29 +DATA ·staticuint64s+0x0f0(SB)/8, $30 +DATA ·staticuint64s+0x0f8(SB)/8, $31 +DATA ·staticuint64s+0x100(SB)/8, $32 +DATA ·staticuint64s+0x108(SB)/8, $33 +DATA ·staticuint64s+0x110(SB)/8, $34 +DATA ·staticuint64s+0x118(SB)/8, $35 +DATA ·staticuint64s+0x120(SB)/8, $36 +DATA ·staticuint64s+0x128(SB)/8, $37 +DATA ·staticuint64s+0x130(SB)/8, $38 +DATA ·staticuint64s+0x138(SB)/8, $39 +DATA ·staticuint64s+0x140(SB)/8, $40 +DATA ·staticuint64s+0x148(SB)/8, $41 +DATA ·staticuint64s+0x150(SB)/8, $42 +DATA ·staticuint64s+0x158(SB)/8, $43 +DATA ·staticuint64s+0x160(SB)/8, $44 +DATA ·staticuint64s+0x168(SB)/8, $45 +DATA ·staticuint64s+0x170(SB)/8, $46 +DATA ·staticuint64s+0x178(SB)/8, $47 +DATA ·staticuint64s+0x180(SB)/8, $48 +DATA ·staticuint64s+0x188(SB)/8, $49 +DATA ·staticuint64s+0x190(SB)/8, $50 +DATA ·staticuint64s+0x198(SB)/8, $51 +DATA ·staticuint64s+0x1a0(SB)/8, $52 +DATA ·staticuint64s+0x1a8(SB)/8, $53 +DATA ·staticuint64s+0x1b0(SB)/8, $54 +DATA ·staticuint64s+0x1b8(SB)/8, $55 +DATA ·staticuint64s+0x1c0(SB)/8, $56 +DATA ·staticuint64s+0x1c8(SB)/8, $57 +DATA ·staticuint64s+0x1d0(SB)/8, $58 +DATA ·staticuint64s+0x1d8(SB)/8, $59 +DATA ·staticuint64s+0x1e0(SB)/8, $60 +DATA ·staticuint64s+0x1e8(SB)/8, $61 +DATA ·staticuint64s+0x1f0(SB)/8, $62 +DATA ·staticuint64s+0x1f8(SB)/8, $63 +DATA ·staticuint64s+0x200(SB)/8, $64 +DATA ·staticuint64s+0x208(SB)/8, $65 +DATA ·staticuint64s+0x210(SB)/8, $66 +DATA ·staticuint64s+0x218(SB)/8, $67 +DATA ·staticuint64s+0x220(SB)/8, $68 +DATA ·staticuint64s+0x228(SB)/8, $69 +DATA ·staticuint64s+0x230(SB)/8, $70 +DATA ·staticuint64s+0x238(SB)/8, $71 +DATA ·staticuint64s+0x240(SB)/8, $72 +DATA ·staticuint64s+0x248(SB)/8, $73 +DATA ·staticuint64s+0x250(SB)/8, $74 +DATA ·staticuint64s+0x258(SB)/8, $75 +DATA ·staticuint64s+0x260(SB)/8, $76 +DATA ·staticuint64s+0x268(SB)/8, $77 +DATA ·staticuint64s+0x270(SB)/8, $78 +DATA ·staticuint64s+0x278(SB)/8, $79 +DATA ·staticuint64s+0x280(SB)/8, $80 +DATA ·staticuint64s+0x288(SB)/8, $81 +DATA ·staticuint64s+0x290(SB)/8, $82 +DATA ·staticuint64s+0x298(SB)/8, $83 +DATA ·staticuint64s+0x2a0(SB)/8, $84 +DATA ·staticuint64s+0x2a8(SB)/8, $85 +DATA ·staticuint64s+0x2b0(SB)/8, $86 +DATA ·staticuint64s+0x2b8(SB)/8, $87 +DATA ·staticuint64s+0x2c0(SB)/8, $88 +DATA ·staticuint64s+0x2c8(SB)/8, $89 +DATA ·staticuint64s+0x2d0(SB)/8, $90 +DATA ·staticuint64s+0x2d8(SB)/8, $91 +DATA ·staticuint64s+0x2e0(SB)/8, $92 +DATA ·staticuint64s+0x2e8(SB)/8, $93 +DATA ·staticuint64s+0x2f0(SB)/8, $94 +DATA ·staticuint64s+0x2f8(SB)/8, $95 +DATA ·staticuint64s+0x300(SB)/8, $96 +DATA ·staticuint64s+0x308(SB)/8, $97 +DATA ·staticuint64s+0x310(SB)/8, $98 +DATA ·staticuint64s+0x318(SB)/8, $99 +DATA ·staticuint64s+0x320(SB)/8, $100 +DATA ·staticuint64s+0x328(SB)/8, $101 +DATA ·staticuint64s+0x330(SB)/8, $102 +DATA ·staticuint64s+0x338(SB)/8, $103 +DATA ·staticuint64s+0x340(SB)/8, $104 +DATA ·staticuint64s+0x348(SB)/8, $105 +DATA ·staticuint64s+0x350(SB)/8, $106 +DATA ·staticuint64s+0x358(SB)/8, $107 +DATA ·staticuint64s+0x360(SB)/8, $108 +DATA ·staticuint64s+0x368(SB)/8, $109 +DATA ·staticuint64s+0x370(SB)/8, $110 +DATA ·staticuint64s+0x378(SB)/8, $111 +DATA ·staticuint64s+0x380(SB)/8, $112 +DATA ·staticuint64s+0x388(SB)/8, $113 +DATA ·staticuint64s+0x390(SB)/8, $114 +DATA ·staticuint64s+0x398(SB)/8, $115 +DATA ·staticuint64s+0x3a0(SB)/8, $116 +DATA ·staticuint64s+0x3a8(SB)/8, $117 +DATA ·staticuint64s+0x3b0(SB)/8, $118 +DATA ·staticuint64s+0x3b8(SB)/8, $119 +DATA ·staticuint64s+0x3c0(SB)/8, $120 +DATA ·staticuint64s+0x3c8(SB)/8, $121 +DATA ·staticuint64s+0x3d0(SB)/8, $122 +DATA ·staticuint64s+0x3d8(SB)/8, $123 +DATA ·staticuint64s+0x3e0(SB)/8, $124 +DATA ·staticuint64s+0x3e8(SB)/8, $125 +DATA ·staticuint64s+0x3f0(SB)/8, $126 +DATA ·staticuint64s+0x3f8(SB)/8, $127 +DATA ·staticuint64s+0x400(SB)/8, $128 +DATA ·staticuint64s+0x408(SB)/8, $129 +DATA ·staticuint64s+0x410(SB)/8, $130 +DATA ·staticuint64s+0x418(SB)/8, $131 +DATA ·staticuint64s+0x420(SB)/8, $132 +DATA ·staticuint64s+0x428(SB)/8, $133 +DATA ·staticuint64s+0x430(SB)/8, $134 +DATA ·staticuint64s+0x438(SB)/8, $135 +DATA ·staticuint64s+0x440(SB)/8, $136 +DATA ·staticuint64s+0x448(SB)/8, $137 +DATA ·staticuint64s+0x450(SB)/8, $138 +DATA ·staticuint64s+0x458(SB)/8, $139 +DATA ·staticuint64s+0x460(SB)/8, $140 +DATA ·staticuint64s+0x468(SB)/8, $141 +DATA ·staticuint64s+0x470(SB)/8, $142 +DATA ·staticuint64s+0x478(SB)/8, $143 +DATA ·staticuint64s+0x480(SB)/8, $144 +DATA ·staticuint64s+0x488(SB)/8, $145 +DATA ·staticuint64s+0x490(SB)/8, $146 +DATA ·staticuint64s+0x498(SB)/8, $147 +DATA ·staticuint64s+0x4a0(SB)/8, $148 +DATA ·staticuint64s+0x4a8(SB)/8, $149 +DATA ·staticuint64s+0x4b0(SB)/8, $150 +DATA ·staticuint64s+0x4b8(SB)/8, $151 +DATA ·staticuint64s+0x4c0(SB)/8, $152 +DATA ·staticuint64s+0x4c8(SB)/8, $153 +DATA ·staticuint64s+0x4d0(SB)/8, $154 +DATA ·staticuint64s+0x4d8(SB)/8, $155 +DATA ·staticuint64s+0x4e0(SB)/8, $156 +DATA ·staticuint64s+0x4e8(SB)/8, $157 +DATA ·staticuint64s+0x4f0(SB)/8, $158 +DATA ·staticuint64s+0x4f8(SB)/8, $159 +DATA ·staticuint64s+0x500(SB)/8, $160 +DATA ·staticuint64s+0x508(SB)/8, $161 +DATA ·staticuint64s+0x510(SB)/8, $162 +DATA ·staticuint64s+0x518(SB)/8, $163 +DATA ·staticuint64s+0x520(SB)/8, $164 +DATA ·staticuint64s+0x528(SB)/8, $165 +DATA ·staticuint64s+0x530(SB)/8, $166 +DATA ·staticuint64s+0x538(SB)/8, $167 +DATA ·staticuint64s+0x540(SB)/8, $168 +DATA ·staticuint64s+0x548(SB)/8, $169 +DATA ·staticuint64s+0x550(SB)/8, $170 +DATA ·staticuint64s+0x558(SB)/8, $171 +DATA ·staticuint64s+0x560(SB)/8, $172 +DATA ·staticuint64s+0x568(SB)/8, $173 +DATA ·staticuint64s+0x570(SB)/8, $174 +DATA ·staticuint64s+0x578(SB)/8, $175 +DATA ·staticuint64s+0x580(SB)/8, $176 +DATA ·staticuint64s+0x588(SB)/8, $177 +DATA ·staticuint64s+0x590(SB)/8, $178 +DATA ·staticuint64s+0x598(SB)/8, $179 +DATA ·staticuint64s+0x5a0(SB)/8, $180 +DATA ·staticuint64s+0x5a8(SB)/8, $181 +DATA ·staticuint64s+0x5b0(SB)/8, $182 +DATA ·staticuint64s+0x5b8(SB)/8, $183 +DATA ·staticuint64s+0x5c0(SB)/8, $184 +DATA ·staticuint64s+0x5c8(SB)/8, $185 +DATA ·staticuint64s+0x5d0(SB)/8, $186 +DATA ·staticuint64s+0x5d8(SB)/8, $187 +DATA ·staticuint64s+0x5e0(SB)/8, $188 +DATA ·staticuint64s+0x5e8(SB)/8, $189 +DATA ·staticuint64s+0x5f0(SB)/8, $190 +DATA ·staticuint64s+0x5f8(SB)/8, $191 +DATA ·staticuint64s+0x600(SB)/8, $192 +DATA ·staticuint64s+0x608(SB)/8, $193 +DATA ·staticuint64s+0x610(SB)/8, $194 +DATA ·staticuint64s+0x618(SB)/8, $195 +DATA ·staticuint64s+0x620(SB)/8, $196 +DATA ·staticuint64s+0x628(SB)/8, $197 +DATA ·staticuint64s+0x630(SB)/8, $198 +DATA ·staticuint64s+0x638(SB)/8, $199 +DATA ·staticuint64s+0x640(SB)/8, $200 +DATA ·staticuint64s+0x648(SB)/8, $201 +DATA ·staticuint64s+0x650(SB)/8, $202 +DATA ·staticuint64s+0x658(SB)/8, $203 +DATA ·staticuint64s+0x660(SB)/8, $204 +DATA ·staticuint64s+0x668(SB)/8, $205 +DATA ·staticuint64s+0x670(SB)/8, $206 +DATA ·staticuint64s+0x678(SB)/8, $207 +DATA ·staticuint64s+0x680(SB)/8, $208 +DATA ·staticuint64s+0x688(SB)/8, $209 +DATA ·staticuint64s+0x690(SB)/8, $210 +DATA ·staticuint64s+0x698(SB)/8, $211 +DATA ·staticuint64s+0x6a0(SB)/8, $212 +DATA ·staticuint64s+0x6a8(SB)/8, $213 +DATA ·staticuint64s+0x6b0(SB)/8, $214 +DATA ·staticuint64s+0x6b8(SB)/8, $215 +DATA ·staticuint64s+0x6c0(SB)/8, $216 +DATA ·staticuint64s+0x6c8(SB)/8, $217 +DATA ·staticuint64s+0x6d0(SB)/8, $218 +DATA ·staticuint64s+0x6d8(SB)/8, $219 +DATA ·staticuint64s+0x6e0(SB)/8, $220 +DATA ·staticuint64s+0x6e8(SB)/8, $221 +DATA ·staticuint64s+0x6f0(SB)/8, $222 +DATA ·staticuint64s+0x6f8(SB)/8, $223 +DATA ·staticuint64s+0x700(SB)/8, $224 +DATA ·staticuint64s+0x708(SB)/8, $225 +DATA ·staticuint64s+0x710(SB)/8, $226 +DATA ·staticuint64s+0x718(SB)/8, $227 +DATA ·staticuint64s+0x720(SB)/8, $228 +DATA ·staticuint64s+0x728(SB)/8, $229 +DATA ·staticuint64s+0x730(SB)/8, $230 +DATA ·staticuint64s+0x738(SB)/8, $231 +DATA ·staticuint64s+0x740(SB)/8, $232 +DATA ·staticuint64s+0x748(SB)/8, $233 +DATA ·staticuint64s+0x750(SB)/8, $234 +DATA ·staticuint64s+0x758(SB)/8, $235 +DATA ·staticuint64s+0x760(SB)/8, $236 +DATA ·staticuint64s+0x768(SB)/8, $237 +DATA ·staticuint64s+0x770(SB)/8, $238 +DATA ·staticuint64s+0x778(SB)/8, $239 +DATA ·staticuint64s+0x780(SB)/8, $240 +DATA ·staticuint64s+0x788(SB)/8, $241 +DATA ·staticuint64s+0x790(SB)/8, $242 +DATA ·staticuint64s+0x798(SB)/8, $243 +DATA ·staticuint64s+0x7a0(SB)/8, $244 +DATA ·staticuint64s+0x7a8(SB)/8, $245 +DATA ·staticuint64s+0x7b0(SB)/8, $246 +DATA ·staticuint64s+0x7b8(SB)/8, $247 +DATA ·staticuint64s+0x7c0(SB)/8, $248 +DATA ·staticuint64s+0x7c8(SB)/8, $249 +DATA ·staticuint64s+0x7d0(SB)/8, $250 +DATA ·staticuint64s+0x7d8(SB)/8, $251 +DATA ·staticuint64s+0x7e0(SB)/8, $252 +DATA ·staticuint64s+0x7e8(SB)/8, $253 +DATA ·staticuint64s+0x7f0(SB)/8, $254 +DATA ·staticuint64s+0x7f8(SB)/8, $255 + +GLOBL ·staticuint64s(SB), RODATA, $0x800 diff --git a/src/runtime/linkname.go b/src/runtime/linkname.go index dd7f6742..eec190c3 100644 --- a/src/runtime/linkname.go +++ b/src/runtime/linkname.go @@ -13,6 +13,7 @@ import _ "unsafe" //go:linkname _cgo_panic_internal //go:linkname cgoAlwaysFalse //go:linkname cgoUse +//go:linkname cgoKeepAlive //go:linkname cgoCheckPointer //go:linkname cgoCheckResult //go:linkname cgoNoCallback diff --git a/src/runtime/linkname_swiss.go b/src/runtime/linkname_swiss.go new file mode 100644 index 00000000..1be72447 --- /dev/null +++ b/src/runtime/linkname_swiss.go @@ -0,0 +1,211 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package runtime + +import ( + "internal/abi" + "internal/runtime/maps" + "internal/runtime/sys" + "unsafe" +) + +// Legacy //go:linkname compatibility shims +// +// The functions below are unused by the toolchain, and exist only for +// compatibility with existing //go:linkname use in the ecosystem (and in +// map_noswiss.go for normal use via GOEXPERIMENT=noswissmap). + +// linknameIter is the it argument to mapiterinit and mapiternext. +// +// Callers of mapiterinit allocate their own iter structure, which has the +// layout of the pre-Go 1.24 hiter structure, shown here for posterity: +// +// type hiter struct { +// key unsafe.Pointer +// elem unsafe.Pointer +// t *maptype +// h *hmap +// buckets unsafe.Pointer +// bptr *bmap +// overflow *[]*bmap +// oldoverflow *[]*bmap +// startBucket uintptr +// offset uint8 +// wrapped bool +// B uint8 +// i uint8 +// bucket uintptr +// checkBucket uintptr +// } +// +// Our structure must maintain compatibility with the old structure. This +// means: +// +// - Our structure must be the same size or smaller than hiter. Otherwise we +// may write outside the caller's hiter allocation. +// - Our structure must have the same pointer layout as hiter, so that the GC +// tracks pointers properly. +// +// Based on analysis of the "hall of shame" users of these linknames: +// +// - The key and elem fields must be kept up to date with the current key/elem. +// Some users directly access the key and elem fields rather than calling +// reflect.mapiterkey/reflect.mapiterelem. +// - The t field must be non-nil after mapiterinit. gonum.org/v1/gonum uses +// this to verify the iterator is initialized. +// - github.com/segmentio/encoding and github.com/RomiChan/protobuf check if h +// is non-nil, but the code has no effect. Thus the value of h does not +// matter. See internal/runtime_reflect/map.go. +type linknameIter struct { + // Fields from hiter. + key unsafe.Pointer + elem unsafe.Pointer + typ *abi.SwissMapType + + // The real iterator. + it *maps.Iter +} + +// mapiterinit is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// mapiterinit should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/goccy/go-json +// - github.com/RomiChan/protobuf +// - github.com/segmentio/encoding +// - github.com/ugorji/go/codec +// - github.com/wI2L/jettison +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapiterinit +func mapiterinit(t *abi.SwissMapType, m *maps.Map, it *linknameIter) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(mapiterinit)) + } + + it.typ = t + + it.it = new(maps.Iter) + it.it.Init(t, m) + it.it.Next() + + it.key = it.it.Key() + it.elem = it.it.Elem() +} + +// reflect_mapiterinit is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiterinit should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/modern-go/reflect2 +// - gitee.com/quant1x/gox +// - github.com/v2pro/plz +// - github.com/wI2L/jettison +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiterinit reflect.mapiterinit +func reflect_mapiterinit(t *abi.SwissMapType, m *maps.Map, it *linknameIter) { + mapiterinit(t, m, it) +} + +// mapiternext is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// mapiternext should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/RomiChan/protobuf +// - github.com/segmentio/encoding +// - github.com/ugorji/go/codec +// - gonum.org/v1/gonum +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapiternext +func mapiternext(it *linknameIter) { + if raceenabled { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(it.it.Map()), callerpc, abi.FuncPCABIInternal(mapiternext)) + } + + it.it.Next() + + it.key = it.it.Key() + it.elem = it.it.Elem() +} + +// reflect_mapiternext is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiternext is for package reflect, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/modern-go/reflect2 +// - github.com/goccy/go-json +// - github.com/v2pro/plz +// - github.com/wI2L/jettison +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiternext reflect.mapiternext +func reflect_mapiternext(it *linknameIter) { + mapiternext(it) +} + +// reflect_mapiterkey is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiterkey should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/goccy/go-json +// - gonum.org/v1/gonum +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiterkey reflect.mapiterkey +func reflect_mapiterkey(it *linknameIter) unsafe.Pointer { + return it.it.Key() +} + +// reflect_mapiterelem is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiterelem should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/goccy/go-json +// - gonum.org/v1/gonum +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiterelem reflect.mapiterelem +func reflect_mapiterelem(it *linknameIter) unsafe.Pointer { + return it.it.Elem() +} diff --git a/src/runtime/lock_futex.go b/src/runtime/lock_futex.go index 58690e45..1cf146b4 100644 --- a/src/runtime/lock_futex.go +++ b/src/runtime/lock_futex.go @@ -11,32 +11,6 @@ import ( "unsafe" ) -// This implementation depends on OS-specific implementations of -// -// futexsleep(addr *uint32, val uint32, ns int64) -// Atomically, -// if *addr == val { sleep } -// Might be woken up spuriously; that's allowed. -// Don't sleep longer than ns; ns < 0 means forever. -// -// futexwakeup(addr *uint32, cnt uint32) -// If any procs are sleeping on addr, wake up at most cnt. - -const ( - mutex_unlocked = 0 - mutex_locked = 1 - mutex_sleeping = 2 - - active_spin = 4 - active_spin_cnt = 30 - passive_spin = 1 -) - -// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping. -// mutex_sleeping means that there is presumably at least one sleeping thread. -// Note that there can be spinning threads during all states - they do not -// affect mutex's state. - // We use the uintptr mutex.key and note.key as a uint32. // //go:nosplit @@ -44,103 +18,6 @@ func key32(p *uintptr) *uint32 { return (*uint32)(unsafe.Pointer(p)) } -func mutexContended(l *mutex) bool { - return atomic.Load(key32(&l.key)) > mutex_locked -} - -func lock(l *mutex) { - lockWithRank(l, getLockRank(l)) -} - -func lock2(l *mutex) { - gp := getg() - - if gp.m.locks < 0 { - throw("runtime·lock: lock count") - } - gp.m.locks++ - - // Speculative grab for lock. - v := atomic.Xchg(key32(&l.key), mutex_locked) - if v == mutex_unlocked { - return - } - - // wait is either MUTEX_LOCKED or MUTEX_SLEEPING - // depending on whether there is a thread sleeping - // on this mutex. If we ever change l->key from - // MUTEX_SLEEPING to some other value, we must be - // careful to change it back to MUTEX_SLEEPING before - // returning, to ensure that the sleeping thread gets - // its wakeup call. - wait := v - - timer := &lockTimer{lock: l} - timer.begin() - // On uniprocessors, no point spinning. - // On multiprocessors, spin for ACTIVE_SPIN attempts. - spin := 0 - if ncpu > 1 { - spin = active_spin - } - for { - // Try for lock, spinning. - for i := 0; i < spin; i++ { - for l.key == mutex_unlocked { - if atomic.Cas(key32(&l.key), mutex_unlocked, wait) { - timer.end() - return - } - } - procyield(active_spin_cnt) - } - - // Try for lock, rescheduling. - for i := 0; i < passive_spin; i++ { - for l.key == mutex_unlocked { - if atomic.Cas(key32(&l.key), mutex_unlocked, wait) { - timer.end() - return - } - } - osyield() - } - - // Sleep. - v = atomic.Xchg(key32(&l.key), mutex_sleeping) - if v == mutex_unlocked { - timer.end() - return - } - wait = mutex_sleeping - futexsleep(key32(&l.key), mutex_sleeping, -1) - } -} - -func unlock(l *mutex) { - unlockWithRank(l) -} - -func unlock2(l *mutex) { - v := atomic.Xchg(key32(&l.key), mutex_unlocked) - if v == mutex_unlocked { - throw("unlock of unlocked lock") - } - if v == mutex_sleeping { - futexwakeup(key32(&l.key), 1) - } - - gp := getg() - gp.m.mLockProfile.recordUnlock(l) - gp.m.locks-- - if gp.m.locks < 0 { - throw("runtime·unlock: lock count") - } - if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack - gp.stackguard0 = stackPreempt - } -} - // One-time notifications. func noteclear(n *note) { n.key = 0 @@ -254,3 +131,33 @@ func beforeIdle(int64, int64) (*g, bool) { } func checkTimeouts() {} + +//go:nosplit +func semacreate(mp *m) {} + +//go:nosplit +func semasleep(ns int64) int32 { + mp := getg().m + + for v := atomic.Xadd(&mp.waitsema, -1); ; v = atomic.Load(&mp.waitsema) { + if int32(v) >= 0 { + return 0 + } + futexsleep(&mp.waitsema, v, ns) + if ns >= 0 { + if int32(v) >= 0 { + return 0 + } else { + return -1 + } + } + } +} + +//go:nosplit +func semawakeup(mp *m) { + v := atomic.Xadd(&mp.waitsema, 1) + if v == 0 { + futexwakeup(&mp.waitsema, 1) + } +} diff --git a/src/runtime/lock_futex_tristate.go b/src/runtime/lock_futex_tristate.go new file mode 100644 index 00000000..b7df18c8 --- /dev/null +++ b/src/runtime/lock_futex_tristate.go @@ -0,0 +1,138 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (dragonfly || freebsd || linux) && !goexperiment.spinbitmutex + +package runtime + +import ( + "internal/runtime/atomic" +) + +// This implementation depends on OS-specific implementations of +// +// futexsleep(addr *uint32, val uint32, ns int64) +// Atomically, +// if *addr == val { sleep } +// Might be woken up spuriously; that's allowed. +// Don't sleep longer than ns; ns < 0 means forever. +// +// futexwakeup(addr *uint32, cnt uint32) +// If any procs are sleeping on addr, wake up at most cnt. + +const ( + mutex_unlocked = 0 + mutex_locked = 1 + mutex_sleeping = 2 + + active_spin = 4 + active_spin_cnt = 30 + passive_spin = 1 +) + +// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping. +// mutex_sleeping means that there is presumably at least one sleeping thread. +// Note that there can be spinning threads during all states - they do not +// affect mutex's state. + +type mWaitList struct{} + +func lockVerifyMSize() {} + +func mutexContended(l *mutex) bool { + return atomic.Load(key32(&l.key)) > mutex_locked +} + +func lock(l *mutex) { + lockWithRank(l, getLockRank(l)) +} + +func lock2(l *mutex) { + gp := getg() + + if gp.m.locks < 0 { + throw("runtime·lock: lock count") + } + gp.m.locks++ + + // Speculative grab for lock. + v := atomic.Xchg(key32(&l.key), mutex_locked) + if v == mutex_unlocked { + return + } + + // wait is either MUTEX_LOCKED or MUTEX_SLEEPING + // depending on whether there is a thread sleeping + // on this mutex. If we ever change l->key from + // MUTEX_SLEEPING to some other value, we must be + // careful to change it back to MUTEX_SLEEPING before + // returning, to ensure that the sleeping thread gets + // its wakeup call. + wait := v + + timer := &lockTimer{lock: l} + timer.begin() + // On uniprocessors, no point spinning. + // On multiprocessors, spin for ACTIVE_SPIN attempts. + spin := 0 + if ncpu > 1 { + spin = active_spin + } + for { + // Try for lock, spinning. + for i := 0; i < spin; i++ { + for l.key == mutex_unlocked { + if atomic.Cas(key32(&l.key), mutex_unlocked, wait) { + timer.end() + return + } + } + procyield(active_spin_cnt) + } + + // Try for lock, rescheduling. + for i := 0; i < passive_spin; i++ { + for l.key == mutex_unlocked { + if atomic.Cas(key32(&l.key), mutex_unlocked, wait) { + timer.end() + return + } + } + osyield() + } + + // Sleep. + v = atomic.Xchg(key32(&l.key), mutex_sleeping) + if v == mutex_unlocked { + timer.end() + return + } + wait = mutex_sleeping + futexsleep(key32(&l.key), mutex_sleeping, -1) + } +} + +func unlock(l *mutex) { + unlockWithRank(l) +} + +func unlock2(l *mutex) { + v := atomic.Xchg(key32(&l.key), mutex_unlocked) + if v == mutex_unlocked { + throw("unlock of unlocked lock") + } + if v == mutex_sleeping { + futexwakeup(key32(&l.key), 1) + } + + gp := getg() + gp.m.mLockProfile.recordUnlock(l) + gp.m.locks-- + if gp.m.locks < 0 { + throw("runtime·unlock: lock count") + } + if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack + gp.stackguard0 = stackPreempt + } +} diff --git a/src/runtime/lock_js.go b/src/runtime/lock_js.go index b6ee5ec7..a40e3010 100644 --- a/src/runtime/lock_js.go +++ b/src/runtime/lock_js.go @@ -6,7 +6,10 @@ package runtime -import _ "unsafe" // for go:linkname +import ( + "internal/runtime/sys" + _ "unsafe" // for go:linkname +) // js/wasm has no support for threads yet. There is no preemption. @@ -23,6 +26,10 @@ const ( passive_spin = 1 ) +type mWaitList struct{} + +func lockVerifyMSize() {} + func mutexContended(l *mutex) bool { return false } @@ -63,29 +70,21 @@ func unlock2(l *mutex) { // One-time notifications. -type noteWithTimeout struct { - gp *g - deadline int64 -} - -var ( - notes = make(map[*note]*g) - notesWithTimeout = make(map[*note]noteWithTimeout) -) +// Linked list of notes with a deadline. +var allDeadlineNotes *note func noteclear(n *note) { - n.key = note_cleared + n.status = note_cleared } func notewakeup(n *note) { - // gp := getg() - if n.key == note_woken { + if n.status == note_woken { throw("notewakeup - double wakeup") } - cleared := n.key == note_cleared - n.key = note_woken + cleared := n.status == note_cleared + n.status = note_woken if cleared { - goready(notes[n], 1) + goready(n.gp, 1) } } @@ -113,48 +112,50 @@ func notetsleepg(n *note, ns int64) bool { } id := scheduleTimeoutEvent(delay) - mp := acquirem() - notes[n] = gp - notesWithTimeout[n] = noteWithTimeout{gp: gp, deadline: deadline} - releasem(mp) + + n.gp = gp + n.deadline = deadline + if allDeadlineNotes != nil { + allDeadlineNotes.allprev = n + } + n.allnext = allDeadlineNotes + allDeadlineNotes = n gopark(nil, nil, waitReasonSleep, traceBlockSleep, 1) clearTimeoutEvent(id) // note might have woken early, clear timeout - mp = acquirem() - delete(notes, n) - delete(notesWithTimeout, n) - releasem(mp) + n.gp = nil + n.deadline = 0 + if n.allprev != nil { + n.allprev.allnext = n.allnext + } + if allDeadlineNotes == n { + allDeadlineNotes = n.allnext + } + n.allprev = nil + n.allnext = nil - return n.key == note_woken + return n.status == note_woken } - for n.key != note_woken { - mp := acquirem() - notes[n] = gp - releasem(mp) + for n.status != note_woken { + n.gp = gp gopark(nil, nil, waitReasonZero, traceBlockGeneric, 1) - mp = acquirem() - delete(notes, n) - releasem(mp) + n.gp = nil } return true } // checkTimeouts resumes goroutines that are waiting on a note which has reached its deadline. -// TODO(drchase): need to understand if write barriers are really okay in this context. -// -//go:yeswritebarrierrec func checkTimeouts() { now := nanotime() - // TODO: map iteration has the write barriers in it; is that okay? - for n, nt := range notesWithTimeout { - if n.key == note_cleared && now >= nt.deadline { - n.key = note_timeout - goready(nt.gp, 1) + for n := allDeadlineNotes; n != nil; n = n.allnext { + if n.status == note_cleared && n.deadline != 0 && now >= n.deadline { + n.status = note_timeout + goready(n.gp, 1) } } } @@ -250,7 +251,7 @@ var idleStart int64 func handleAsyncEvent() { idleStart = nanotime() - pause(getcallersp() - 16) + pause(sys.GetCallerSP() - 16) } // clearIdleTimeout clears our record of the timeout started by beforeIdle. @@ -259,9 +260,6 @@ func clearIdleTimeout() { idleTimeout = nil } -// pause sets SP to newsp and pauses the execution of Go's WebAssembly code until an event is triggered. -func pause(newsp uintptr) - // scheduleTimeoutEvent tells the WebAssembly environment to trigger an event after ms milliseconds. // It returns a timer id that can be used with clearTimeoutEvent. // @@ -300,7 +298,7 @@ func handleEvent() { // return execution to JavaScript idleStart = nanotime() - pause(getcallersp() - 16) + pause(sys.GetCallerSP() - 16) } // eventHandler retrieves and executes handlers for pending JavaScript events. diff --git a/src/runtime/lock_sema.go b/src/runtime/lock_sema.go index 32d2235a..3e1b07b9 100644 --- a/src/runtime/lock_sema.go +++ b/src/runtime/lock_sema.go @@ -11,131 +11,10 @@ import ( "unsafe" ) -// This implementation depends on OS-specific implementations of -// -// func semacreate(mp *m) -// Create a semaphore for mp, if it does not already have one. -// -// func semasleep(ns int64) int32 -// If ns < 0, acquire m's semaphore and return 0. -// If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds. -// Return 0 if the semaphore was acquired, -1 if interrupted or timed out. -// -// func semawakeup(mp *m) -// Wake up mp, which is or will soon be sleeping on its semaphore. const ( locked uintptr = 1 - - active_spin = 4 - active_spin_cnt = 30 - passive_spin = 1 ) -func mutexContended(l *mutex) bool { - return atomic.Loaduintptr(&l.key) > locked -} - -func lock(l *mutex) { - lockWithRank(l, getLockRank(l)) -} - -func lock2(l *mutex) { - gp := getg() - if gp.m.locks < 0 { - throw("runtime·lock: lock count") - } - gp.m.locks++ - - // Speculative grab for lock. - if atomic.Casuintptr(&l.key, 0, locked) { - return - } - semacreate(gp.m) - - timer := &lockTimer{lock: l} - timer.begin() - // On uniprocessor's, no point spinning. - // On multiprocessors, spin for ACTIVE_SPIN attempts. - spin := 0 - if ncpu > 1 { - spin = active_spin - } -Loop: - for i := 0; ; i++ { - v := atomic.Loaduintptr(&l.key) - if v&locked == 0 { - // Unlocked. Try to lock. - if atomic.Casuintptr(&l.key, v, v|locked) { - timer.end() - return - } - i = 0 - } - if i < spin { - procyield(active_spin_cnt) - } else if i < spin+passive_spin { - osyield() - } else { - // Someone else has it. - // l->waitm points to a linked list of M's waiting - // for this lock, chained through m->nextwaitm. - // Queue this M. - for { - gp.m.nextwaitm = muintptr(v &^ locked) - if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) { - break - } - v = atomic.Loaduintptr(&l.key) - if v&locked == 0 { - continue Loop - } - } - if v&locked != 0 { - // Queued. Wait. - semasleep(-1) - i = 0 - } - } - } -} - -func unlock(l *mutex) { - unlockWithRank(l) -} - -// We might not be holding a p in this code. -// -//go:nowritebarrier -func unlock2(l *mutex) { - gp := getg() - var mp *m - for { - v := atomic.Loaduintptr(&l.key) - if v == locked { - if atomic.Casuintptr(&l.key, locked, 0) { - break - } - } else { - // Other M's are waiting for the lock. - // Dequeue an M. - mp = muintptr(v &^ locked).ptr() - if atomic.Casuintptr(&l.key, v, uintptr(mp.nextwaitm)) { - // Dequeued an M. Wake it. - semawakeup(mp) - break - } - } - } - gp.m.mLockProfile.recordUnlock(l) - gp.m.locks-- - if gp.m.locks < 0 { - throw("runtime·unlock: lock count") - } - if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack - gp.stackguard0 = stackPreempt - } -} - // One-time notifications. func noteclear(n *note) { n.key = 0 diff --git a/src/runtime/lock_sema_tristate.go b/src/runtime/lock_sema_tristate.go new file mode 100644 index 00000000..4375791d --- /dev/null +++ b/src/runtime/lock_sema_tristate.go @@ -0,0 +1,148 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (aix || darwin || netbsd || openbsd || plan9 || solaris || windows) && !goexperiment.spinbitmutex + +package runtime + +import ( + "internal/runtime/atomic" + "unsafe" +) + +// This implementation depends on OS-specific implementations of +// +// func semacreate(mp *m) +// Create a semaphore for mp, if it does not already have one. +// +// func semasleep(ns int64) int32 +// If ns < 0, acquire m's semaphore and return 0. +// If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds. +// Return 0 if the semaphore was acquired, -1 if interrupted or timed out. +// +// func semawakeup(mp *m) +// Wake up mp, which is or will soon be sleeping on its semaphore. +const ( + active_spin = 4 + active_spin_cnt = 30 + passive_spin = 1 +) + +// mWaitList is part of the M struct, and holds the list of Ms that are waiting +// for a particular runtime.mutex. +// +// When an M is unable to immediately obtain a lock, it adds itself to the list +// of Ms waiting for the lock. It does that via this struct's next field, +// forming a singly-linked list with the mutex's key field pointing to the head +// of the list. +type mWaitList struct { + next muintptr // next m waiting for lock +} + +func lockVerifyMSize() {} + +func mutexContended(l *mutex) bool { + return atomic.Loaduintptr(&l.key) > locked +} + +func lock(l *mutex) { + lockWithRank(l, getLockRank(l)) +} + +func lock2(l *mutex) { + gp := getg() + if gp.m.locks < 0 { + throw("runtime·lock: lock count") + } + gp.m.locks++ + + // Speculative grab for lock. + if atomic.Casuintptr(&l.key, 0, locked) { + return + } + semacreate(gp.m) + + timer := &lockTimer{lock: l} + timer.begin() + // On uniprocessor's, no point spinning. + // On multiprocessors, spin for ACTIVE_SPIN attempts. + spin := 0 + if ncpu > 1 { + spin = active_spin + } +Loop: + for i := 0; ; i++ { + v := atomic.Loaduintptr(&l.key) + if v&locked == 0 { + // Unlocked. Try to lock. + if atomic.Casuintptr(&l.key, v, v|locked) { + timer.end() + return + } + i = 0 + } + if i < spin { + procyield(active_spin_cnt) + } else if i < spin+passive_spin { + osyield() + } else { + // Someone else has it. + // l.key points to a linked list of M's waiting + // for this lock, chained through m.mWaitList.next. + // Queue this M. + for { + gp.m.mWaitList.next = muintptr(v &^ locked) + if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) { + break + } + v = atomic.Loaduintptr(&l.key) + if v&locked == 0 { + continue Loop + } + } + if v&locked != 0 { + // Queued. Wait. + semasleep(-1) + i = 0 + } + } + } +} + +func unlock(l *mutex) { + unlockWithRank(l) +} + +// We might not be holding a p in this code. +// +//go:nowritebarrier +func unlock2(l *mutex) { + gp := getg() + var mp *m + for { + v := atomic.Loaduintptr(&l.key) + if v == locked { + if atomic.Casuintptr(&l.key, locked, 0) { + break + } + } else { + // Other M's are waiting for the lock. + // Dequeue an M. + mp = muintptr(v &^ locked).ptr() + if atomic.Casuintptr(&l.key, v, uintptr(mp.mWaitList.next)) { + // Dequeued an M. Wake it. + semawakeup(mp) // no use of mp after this point; it's awake + break + } + } + } + gp.m.mLockProfile.recordUnlock(l) + gp.m.locks-- + if gp.m.locks < 0 { + throw("runtime·unlock: lock count") + } + if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack + gp.stackguard0 = stackPreempt + } +} diff --git a/src/runtime/lock_spinbit.go b/src/runtime/lock_spinbit.go new file mode 100644 index 00000000..7e84f3e1 --- /dev/null +++ b/src/runtime/lock_spinbit.go @@ -0,0 +1,372 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || plan9 || solaris || windows) && goexperiment.spinbitmutex + +package runtime + +import ( + "internal/goarch" + "internal/runtime/atomic" + "unsafe" +) + +// This implementation depends on OS-specific implementations of +// +// func semacreate(mp *m) +// Create a semaphore for mp, if it does not already have one. +// +// func semasleep(ns int64) int32 +// If ns < 0, acquire m's semaphore and return 0. +// If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds. +// Return 0 if the semaphore was acquired, -1 if interrupted or timed out. +// +// func semawakeup(mp *m) +// Wake up mp, which is or will soon be sleeping on its semaphore. + +// The mutex state consists of four flags and a pointer. The flag at bit 0, +// mutexLocked, represents the lock itself. Bit 1, mutexSleeping, is a hint that +// the pointer is non-nil. The fast paths for locking and unlocking the mutex +// are based on atomic 8-bit swap operations on the low byte; bits 2 through 7 +// are unused. +// +// Bit 8, mutexSpinning, is a try-lock that grants a waiting M permission to +// spin on the state word. Most other Ms must attempt to spend their time +// sleeping to reduce traffic on the cache line. This is the "spin bit" for +// which the implementation is named. (The anti-starvation mechanism also grants +// temporary permission for an M to spin.) +// +// Bit 9, mutexStackLocked, is a try-lock that grants an unlocking M permission +// to inspect the list of waiting Ms and to pop an M off of that stack. +// +// The upper bits hold a (partial) pointer to the M that most recently went to +// sleep. The sleeping Ms form a stack linked by their mWaitList.next fields. +// Because the fast paths use an 8-bit swap on the low byte of the state word, +// we'll need to reconstruct the full M pointer from the bits we have. Most Ms +// are allocated on the heap, and have a known alignment and base offset. (The +// offset is due to mallocgc's allocation headers.) The main program thread uses +// a static M value, m0. We check for m0 specifically and add a known offset +// otherwise. + +const ( + active_spin = 4 // referenced in proc.go for sync.Mutex implementation + active_spin_cnt = 30 // referenced in proc.go for sync.Mutex implementation +) + +const ( + mutexLocked = 0x001 + mutexSleeping = 0x002 + mutexSpinning = 0x100 + mutexStackLocked = 0x200 + mutexMMask = 0x3FF + mutexMOffset = mallocHeaderSize // alignment of heap-allocated Ms (those other than m0) + + mutexActiveSpinCount = 4 + mutexActiveSpinSize = 30 + mutexPassiveSpinCount = 1 + + mutexTailWakePeriod = 16 +) + +//go:nosplit +func key8(p *uintptr) *uint8 { + if goarch.BigEndian { + return &(*[8]uint8)(unsafe.Pointer(p))[goarch.PtrSize/1-1] + } + return &(*[8]uint8)(unsafe.Pointer(p))[0] +} + +// mWaitList is part of the M struct, and holds the list of Ms that are waiting +// for a particular runtime.mutex. +// +// When an M is unable to immediately obtain a lock, it adds itself to the list +// of Ms waiting for the lock. It does that via this struct's next field, +// forming a singly-linked list with the mutex's key field pointing to the head +// of the list. +type mWaitList struct { + next muintptr // next m waiting for lock +} + +// lockVerifyMSize confirms that we can recreate the low bits of the M pointer. +func lockVerifyMSize() { + size := roundupsize(unsafe.Sizeof(m{}), false) + mallocHeaderSize + if size&mutexMMask != 0 { + print("M structure uses sizeclass ", size, "/", hex(size), " bytes; ", + "incompatible with mutex flag mask ", hex(mutexMMask), "\n") + throw("runtime.m memory alignment too small for spinbit mutex") + } +} + +// mutexWaitListHead recovers a full muintptr that was missing its low bits. +// With the exception of the static m0 value, it requires allocating runtime.m +// values in a size class with a particular minimum alignment. The 2048-byte +// size class allows recovering the full muintptr value even after overwriting +// the low 11 bits with flags. We can use those 11 bits as 3 flags and an +// atomically-swapped byte. +// +//go:nosplit +func mutexWaitListHead(v uintptr) muintptr { + if highBits := v &^ mutexMMask; highBits == 0 { + return 0 + } else if m0bits := muintptr(unsafe.Pointer(&m0)); highBits == uintptr(m0bits)&^mutexMMask { + return m0bits + } else { + return muintptr(highBits + mutexMOffset) + } +} + +// mutexPreferLowLatency reports if this mutex prefers low latency at the risk +// of performance collapse. If so, we can allow all waiting threads to spin on +// the state word rather than go to sleep. +// +// TODO: We could have the waiting Ms each spin on their own private cache line, +// especially if we can put a bound on the on-CPU time that would consume. +// +// TODO: If there's a small set of mutex values with special requirements, they +// could make use of a more specialized lock2/unlock2 implementation. Otherwise, +// we're constrained to what we can fit within a single uintptr with no +// additional storage on the M for each lock held. +// +//go:nosplit +func mutexPreferLowLatency(l *mutex) bool { + switch l { + default: + return false + case &sched.lock: + // We often expect sched.lock to pass quickly between Ms in a way that + // each M has unique work to do: for instance when we stop-the-world + // (bringing each P to idle) or add new netpoller-triggered work to the + // global run queue. + return true + } +} + +func mutexContended(l *mutex) bool { + return atomic.Loaduintptr(&l.key) > mutexLocked +} + +func lock(l *mutex) { + lockWithRank(l, getLockRank(l)) +} + +func lock2(l *mutex) { + gp := getg() + if gp.m.locks < 0 { + throw("runtime·lock: lock count") + } + gp.m.locks++ + + k8 := key8(&l.key) + + // Speculative grab for lock. + v8 := atomic.Xchg8(k8, mutexLocked) + if v8&mutexLocked == 0 { + if v8&mutexSleeping != 0 { + atomic.Or8(k8, mutexSleeping) + } + return + } + semacreate(gp.m) + + timer := &lockTimer{lock: l} + timer.begin() + // On uniprocessors, no point spinning. + // On multiprocessors, spin for mutexActiveSpinCount attempts. + spin := 0 + if ncpu > 1 { + spin = mutexActiveSpinCount + } + + var weSpin, atTail bool + v := atomic.Loaduintptr(&l.key) +tryAcquire: + for i := 0; ; i++ { + if v&mutexLocked == 0 { + if weSpin { + next := (v &^ mutexSpinning) | mutexSleeping | mutexLocked + if next&^mutexMMask == 0 { + // The fast-path Xchg8 may have cleared mutexSleeping. Fix + // the hint so unlock2 knows when to use its slow path. + next = next &^ mutexSleeping + } + if atomic.Casuintptr(&l.key, v, next) { + timer.end() + return + } + } else { + prev8 := atomic.Xchg8(k8, mutexLocked|mutexSleeping) + if prev8&mutexLocked == 0 { + timer.end() + return + } + } + v = atomic.Loaduintptr(&l.key) + continue tryAcquire + } + + if !weSpin && v&mutexSpinning == 0 && atomic.Casuintptr(&l.key, v, v|mutexSpinning) { + v |= mutexSpinning + weSpin = true + } + + if weSpin || atTail || mutexPreferLowLatency(l) { + if i < spin { + procyield(mutexActiveSpinSize) + v = atomic.Loaduintptr(&l.key) + continue tryAcquire + } else if i < spin+mutexPassiveSpinCount { + osyield() // TODO: Consider removing this step. See https://go.dev/issue/69268. + v = atomic.Loaduintptr(&l.key) + continue tryAcquire + } + } + + // Go to sleep + if v&mutexLocked == 0 { + throw("runtime·lock: sleeping while lock is available") + } + + // Store the current head of the list of sleeping Ms in our gp.m.mWaitList.next field + gp.m.mWaitList.next = mutexWaitListHead(v) + + // Pack a (partial) pointer to this M with the current lock state bits + next := (uintptr(unsafe.Pointer(gp.m)) &^ mutexMMask) | v&mutexMMask | mutexSleeping + if weSpin { // If we were spinning, prepare to retire + next = next &^ mutexSpinning + } + + if atomic.Casuintptr(&l.key, v, next) { + weSpin = false + // We've pushed ourselves onto the stack of waiters. Wait. + semasleep(-1) + atTail = gp.m.mWaitList.next == 0 // we were at risk of starving + i = 0 + } + + gp.m.mWaitList.next = 0 + v = atomic.Loaduintptr(&l.key) + } +} + +func unlock(l *mutex) { + unlockWithRank(l) +} + +// We might not be holding a p in this code. +// +//go:nowritebarrier +func unlock2(l *mutex) { + gp := getg() + + prev8 := atomic.Xchg8(key8(&l.key), 0) + if prev8&mutexLocked == 0 { + throw("unlock of unlocked lock") + } + + if prev8&mutexSleeping != 0 { + unlock2Wake(l) + } + + gp.m.mLockProfile.recordUnlock(l) + gp.m.locks-- + if gp.m.locks < 0 { + throw("runtime·unlock: lock count") + } + if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack + gp.stackguard0 = stackPreempt + } +} + +// unlock2Wake updates the list of Ms waiting on l, waking an M if necessary. +// +//go:nowritebarrier +func unlock2Wake(l *mutex) { + v := atomic.Loaduintptr(&l.key) + + // On occasion, seek out and wake the M at the bottom of the stack so it + // doesn't starve. + antiStarve := cheaprandn(mutexTailWakePeriod) == 0 + if !(antiStarve || // avoiding starvation may require a wake + v&mutexSpinning == 0 || // no spinners means we must wake + mutexPreferLowLatency(l)) { // prefer waiters be awake as much as possible + return + } + + for { + if v&^mutexMMask == 0 || v&mutexStackLocked != 0 { + // No waiting Ms means nothing to do. + // + // If the stack lock is unavailable, its owner would make the same + // wake decisions that we would, so there's nothing for us to do. + // + // Although: This thread may have a different call stack, which + // would result in a different entry in the mutex contention profile + // (upon completion of go.dev/issue/66999). That could lead to weird + // results if a slow critical section ends but another thread + // quickly takes the lock, finishes its own critical section, + // releases the lock, and then grabs the stack lock. That quick + // thread would then take credit (blame) for the delay that this + // slow thread caused. The alternative is to have more expensive + // atomic operations (a CAS) on the critical path of unlock2. + return + } + // Other M's are waiting for the lock. + // Obtain the stack lock, and pop off an M. + next := v | mutexStackLocked + if atomic.Casuintptr(&l.key, v, next) { + break + } + v = atomic.Loaduintptr(&l.key) + } + + // We own the mutexStackLocked flag. New Ms may push themselves onto the + // stack concurrently, but we're now the only thread that can remove or + // modify the Ms that are sleeping in the list. + + var committed *m // If we choose an M within the stack, we've made a promise to wake it + for { + headM := v &^ mutexMMask + flags := v & (mutexMMask &^ mutexStackLocked) // preserve low bits, but release stack lock + + mp := mutexWaitListHead(v).ptr() + wakem := committed + if committed == nil { + if v&mutexSpinning == 0 || mutexPreferLowLatency(l) { + wakem = mp + } + if antiStarve { + // Wake the M at the bottom of the stack of waiters. (This is + // O(N) with the number of waiters.) + wakem = mp + prev := mp + for { + next := wakem.mWaitList.next.ptr() + if next == nil { + break + } + prev, wakem = wakem, next + } + if wakem != mp { + prev.mWaitList.next = wakem.mWaitList.next + committed = wakem + } + } + } + + if wakem == mp { + headM = uintptr(mp.mWaitList.next) &^ mutexMMask + } + + next := headM | flags + if atomic.Casuintptr(&l.key, v, next) { + if wakem != nil { + // Claimed an M. Wake it. + semawakeup(wakem) + } + break + } + + v = atomic.Loaduintptr(&l.key) + } +} diff --git a/src/runtime/lock_wasip1.go b/src/runtime/lock_wasip1.go index acfc62ac..55153c3a 100644 --- a/src/runtime/lock_wasip1.go +++ b/src/runtime/lock_wasip1.go @@ -19,6 +19,10 @@ const ( active_spin_cnt = 30 ) +type mWaitList struct{} + +func lockVerifyMSize() {} + func mutexContended(l *mutex) bool { return false } diff --git a/src/runtime/lockrank.go b/src/runtime/lockrank.go index 37383833..7a5a6185 100644 --- a/src/runtime/lockrank.go +++ b/src/runtime/lockrank.go @@ -43,6 +43,7 @@ const ( lockRankRoot lockRankItab lockRankReflectOffs + lockRankSynctest lockRankUserArenaState // TRACEGLOBAL lockRankTraceBuf @@ -116,6 +117,7 @@ var lockNames = []string{ lockRankRoot: "root", lockRankItab: "itab", lockRankReflectOffs: "reflectOffs", + lockRankSynctest: "synctest", lockRankUserArenaState: "userArenaState", lockRankTraceBuf: "traceBuf", lockRankTraceStrings: "traceStrings", @@ -196,6 +198,7 @@ var lockPartialOrder [][]lockRank = [][]lockRank{ lockRankRoot: {}, lockRankItab: {}, lockRankReflectOffs: {lockRankItab}, + lockRankSynctest: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankRoot, lockRankItab, lockRankReflectOffs}, lockRankUserArenaState: {}, lockRankTraceBuf: {lockRankSysmon, lockRankScavenge}, lockRankTraceStrings: {lockRankSysmon, lockRankScavenge, lockRankTraceBuf}, @@ -208,16 +211,16 @@ var lockPartialOrder [][]lockRank = [][]lockRank{ lockRankProfBlock: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, lockRankProfMemActive: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, lockRankProfMemFuture: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankProfMemActive}, - lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture}, - lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, - lockRankStackLarge: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, - lockRankHchanLeaf: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf}, - lockRankWbufSpans: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, - lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans}, - lockRankMheapSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, - lockRankGlobalAlloc: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankMheapSpecial}, - lockRankTrace: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, - lockRankTraceStackTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace}, + lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture}, + lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, + lockRankStackLarge: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, + lockRankHchanLeaf: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf}, + lockRankWbufSpans: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, + lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans}, + lockRankMheapSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, + lockRankGlobalAlloc: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankMheapSpecial}, + lockRankTrace: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, + lockRankTraceStackTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace}, lockRankPanic: {}, lockRankDeadlock: {lockRankPanic, lockRankDeadlock}, lockRankRaceFini: {lockRankPanic}, diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index b92a2132..73d663f7 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -104,8 +104,8 @@ import ( "internal/goarch" "internal/goos" "internal/runtime/atomic" - "runtime/internal/math" - "runtime/internal/sys" + "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) @@ -227,7 +227,7 @@ const ( // -------------- --------- ---------- ---------- ----------- // */64-bit 48 64MB 1 4M (32MB) // windows/64-bit 48 4MB 64 1M (8MB) - // ios/arm64 33 4MB 1 2048 (8KB) + // ios/arm64 40 4MB 1 256K (2MB) // */32-bit 32 4MB 1 1024 (4KB) // */mips(le) 31 4MB 1 512 (2KB) @@ -427,8 +427,15 @@ func mallocinit() { // Check that the minimum size (exclusive) for a malloc header is also // a size class boundary. This is important to making sure checks align // across different parts of the runtime. + // + // While we're here, also check to make sure all these size classes' + // span sizes are one page. Some code relies on this. minSizeForMallocHeaderIsSizeClass := false + sizeClassesUpToMinSizeForMallocHeaderAreOnePage := true for i := 0; i < len(class_to_size); i++ { + if class_to_allocnpages[i] > 1 { + sizeClassesUpToMinSizeForMallocHeaderAreOnePage = false + } if minSizeForMallocHeader == uintptr(class_to_size[i]) { minSizeForMallocHeaderIsSizeClass = true break @@ -437,6 +444,9 @@ func mallocinit() { if !minSizeForMallocHeaderIsSizeClass { throw("min size of malloc header is not a size class boundary") } + if !sizeClassesUpToMinSizeForMallocHeaderAreOnePage { + throw("expected all size classes up to min size for malloc header to fit in one-page spans") + } // Check that the pointer bitmap for all small sizes without a malloc header // fits in a word. if minSizeForMallocHeader/goarch.PtrSize > 8*goarch.PtrSize { @@ -444,7 +454,7 @@ func mallocinit() { } if minTagBits > taggedPointerBits { - throw("taggedPointerbits too small") + throw("taggedPointerBits too small") } // Initialize the heap. @@ -460,7 +470,10 @@ func mallocinit() { lockInit(&globalAlloc.mutex, lockRankGlobalAlloc) // Create initial arena growth hints. - if goarch.PtrSize == 8 { + if isSbrkPlatform { + // Don't generate hints on sbrk platforms. We can + // only grow the break sequentially. + } else if goarch.PtrSize == 8 { // On a 64-bit machine, we pick the following hints // because: // @@ -818,6 +831,12 @@ mapped: // aligned to align bytes. It may reserve either n or n+align bytes, // so it returns the size that was reserved. func sysReserveAligned(v unsafe.Pointer, size, align uintptr) (unsafe.Pointer, uintptr) { + if isSbrkPlatform { + if v != nil { + throw("unexpected heap arena hint on sbrk platform") + } + return sysReserveAlignedSbrk(size, align) + } // Since the alignment is rather large in uses of this // function, we're not likely to get it by chance, so we ask // for a larger region and remove the parts we don't need. @@ -932,9 +951,9 @@ func nextFreeFast(s *mspan) gclinkptr { // // Must run in a non-preemptible context since otherwise the owner of // c could change. -func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bool) { +func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger bool) { s = c.alloc[spc] - shouldhelpgc = false + checkGCTrigger = false freeIndex := s.nextFreeIndex() if freeIndex == s.nelems { // The span is full. @@ -943,7 +962,7 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bo throw("s.allocCount != s.nelems && freeIndex == s.nelems") } c.refill(spc) - shouldhelpgc = true + checkGCTrigger = true s = c.alloc[spc] freeIndex = s.nextFreeIndex() @@ -962,6 +981,14 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bo return } +// doubleCheckMalloc enables a bunch of extra checks to malloc to double-check +// that various invariants are upheld. +// +// We might consider turning these on by default; many of them previously were. +// They account for a few % of mallocgc's cost though, which does matter somewhat +// at scale. +const doubleCheckMalloc = false + // Allocate an object of size bytes. // Small objects are allocated from the per-P cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. @@ -981,215 +1008,181 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bo // //go:linkname mallocgc func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } } + // Short-circuit zero-sized allocation requests. if size == 0 { return unsafe.Pointer(&zerobase) } // It's possible for any malloc to trigger sweeping, which may in // turn queue finalizers. Record this dynamic lock edge. + // N.B. Compiled away if lockrank experiment is not enabled. lockRankMayQueueFinalizer() - userSize := size - if asanenabled { - // Refer to ASAN runtime library, the malloc() function allocates extra memory, - // the redzone, around the user requested memory region. And the redzones are marked - // as unaddressable. We perform the same operations in Go to detect the overflows or - // underflows. - size += computeRZlog(size) - } - + // Pre-malloc debug hooks. if debug.malloc { - if debug.sbrk != 0 { - align := uintptr(16) - if typ != nil { - // TODO(austin): This should be just - // align = uintptr(typ.align) - // but that's only 4 on 32-bit platforms, - // even if there's a uint64 field in typ (see #599). - // This causes 64-bit atomic accesses to panic. - // Hence, we use stricter alignment that matches - // the normal allocator better. - if size&7 == 0 { - align = 8 - } else if size&3 == 0 { - align = 4 - } else if size&1 == 0 { - align = 2 - } else { - align = 1 - } - } - return persistentalloc(size, align, &memstats.other_sys) - } - - if inittrace.active && inittrace.id == getg().goid { - // Init functions are executed sequentially in a single goroutine. - inittrace.allocs += 1 + if x := preMallocgcDebug(size, typ); x != nil { + return x } } - // assistG is the G to charge for this allocation, or nil if - // GC is not currently active. - assistG := deductAssistCredit(size) + // For ASAN, we allocate extra memory around each allocation called the "redzone." + // These "redzones" are marked as unaddressable. + var asanRZ uintptr + if asanenabled { + asanRZ = redZoneSize(size) + size += asanRZ + } + // Assist the GC if needed. + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + // Actually do the allocation. + var x unsafe.Pointer + var elemsize uintptr + if size <= maxSmallSize-mallocHeaderSize { + if typ == nil || !typ.Pointers() { + if size < maxTinySize { + x, elemsize = mallocgcTiny(size, typ, needzero) + } else { + x, elemsize = mallocgcSmallNoscan(size, typ, needzero) + } + } else if heapBitsInSpan(size) { + x, elemsize = mallocgcSmallScanNoHeader(size, typ, needzero) + } else { + x, elemsize = mallocgcSmallScanHeader(size, typ, needzero) + } + } else { + x, elemsize = mallocgcLarge(size, typ, needzero) + } + + // Notify sanitizers, if enabled. + if raceenabled { + racemalloc(x, size-asanRZ) + } + if msanenabled { + msanmalloc(x, size-asanRZ) + } + if asanenabled { + // Poison the space between the end of the requested size of x + // and the end of the slot. Unpoison the requested allocation. + frag := elemsize - size + if typ != nil && typ.Pointers() && !heapBitsInSpan(elemsize) && size <= maxSmallSize-mallocHeaderSize { + frag -= mallocHeaderSize + } + asanpoison(unsafe.Add(x, size-asanRZ), asanRZ) + asanunpoison(x, size-asanRZ) + } + + // Adjust our GC assist debt to account for internal fragmentation. + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + // Post-malloc debug hooks. + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcTiny(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { // Set mp.mallocing to keep from being preempted by GC. mp := acquirem() - if mp.mallocing != 0 { - throw("malloc deadlock") - } - if mp.gsignal == getg() { - throw("malloc during signal") + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ != nil && typ.Pointers() { + throw("expected noscan for tiny alloc") + } } mp.mallocing = 1 - shouldhelpgc := false - dataSize := userSize - c := getMCache(mp) - if c == nil { - throw("mallocgc called without a P or outside bootstrapping") - } - var span *mspan - var header **_type - var x unsafe.Pointer - noscan := typ == nil || !typ.Pointers() - // In some cases block zeroing can profitably (for latency reduction purposes) - // be delayed till preemption is possible; delayedZeroing tracks that state. - delayedZeroing := false - // Determine if it's a 'small' object that goes into a size-classed span. + // Tiny allocator. // - // Note: This comparison looks a little strange, but it exists to smooth out - // the crossover between the largest size class and large objects that have - // their own spans. The small window of object sizes between maxSmallSize-mallocHeaderSize - // and maxSmallSize will be considered large, even though they might fit in - // a size class. In practice this is completely fine, since the largest small - // size class has a single object in it already, precisely to make the transition - // to large objects smooth. - if size <= maxSmallSize-mallocHeaderSize { - if noscan && size < maxTinySize { - // Tiny allocator. - // - // Tiny allocator combines several tiny allocation requests - // into a single memory block. The resulting memory block - // is freed when all subobjects are unreachable. The subobjects - // must be noscan (don't have pointers), this ensures that - // the amount of potentially wasted memory is bounded. - // - // Size of the memory block used for combining (maxTinySize) is tunable. - // Current setting is 16 bytes, which relates to 2x worst case memory - // wastage (when all but one subobjects are unreachable). - // 8 bytes would result in no wastage at all, but provides less - // opportunities for combining. - // 32 bytes provides more opportunities for combining, - // but can lead to 4x worst case wastage. - // The best case winning is 8x regardless of block size. - // - // Objects obtained from tiny allocator must not be freed explicitly. - // So when an object will be freed explicitly, we ensure that - // its size >= maxTinySize. - // - // SetFinalizer has a special case for objects potentially coming - // from tiny allocator, it such case it allows to set finalizers - // for an inner byte of a memory block. - // - // The main targets of tiny allocator are small strings and - // standalone escaping variables. On a json benchmark - // the allocator reduces number of allocations by ~12% and - // reduces heap size by ~20%. - off := c.tinyoffset - // Align tiny pointer for required (conservative) alignment. - if size&7 == 0 { - off = alignUp(off, 8) - } else if goarch.PtrSize == 4 && size == 12 { - // Conservatively align 12-byte objects to 8 bytes on 32-bit - // systems so that objects whose first field is a 64-bit - // value is aligned to 8 bytes and does not cause a fault on - // atomic access. See issue 37262. - // TODO(mknyszek): Remove this workaround if/when issue 36606 - // is resolved. - off = alignUp(off, 8) - } else if size&3 == 0 { - off = alignUp(off, 4) - } else if size&1 == 0 { - off = alignUp(off, 2) - } - if off+size <= maxTinySize && c.tiny != 0 { - // The object fits into existing tiny block. - x = unsafe.Pointer(c.tiny + off) - c.tinyoffset = off + size - c.tinyAllocs++ - mp.mallocing = 0 - releasem(mp) - return x - } - // Allocate a new maxTinySize block. - span = c.alloc[tinySpanClass] - v := nextFreeFast(span) - if v == 0 { - v, span, shouldhelpgc = c.nextFree(tinySpanClass) - } - x = unsafe.Pointer(v) - (*[2]uint64)(x)[0] = 0 - (*[2]uint64)(x)[1] = 0 - // See if we need to replace the existing tiny block with the new one - // based on amount of remaining free space. - if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { - // Note: disabled when race detector is on, see comment near end of this function. - c.tiny = uintptr(x) - c.tinyoffset = size - } - size = maxTinySize - } else { - hasHeader := !noscan && !heapBitsInSpan(size) - if hasHeader { - size += mallocHeaderSize - } - var sizeclass uint8 - if size <= smallSizeMax-8 { - sizeclass = size_to_class8[divRoundUp(size, smallSizeDiv)] - } else { - sizeclass = size_to_class128[divRoundUp(size-smallSizeMax, largeSizeDiv)] - } - size = uintptr(class_to_size[sizeclass]) - spc := makeSpanClass(sizeclass, noscan) - span = c.alloc[spc] - v := nextFreeFast(span) - if v == 0 { - v, span, shouldhelpgc = c.nextFree(spc) - } - x = unsafe.Pointer(v) - if needzero && span.needzero != 0 { - memclrNoHeapPointers(x, size) - } - if hasHeader { - header = (**_type)(x) - x = add(x, mallocHeaderSize) - size -= mallocHeaderSize - } - } - } else { - shouldhelpgc = true - // For large allocations, keep track of zeroed state so that - // bulk zeroing can be happen later in a preemptible context. - span = c.allocLarge(size, noscan) - span.freeindex = 1 - span.allocCount = 1 - size = span.elemsize - x = unsafe.Pointer(span.base()) - if needzero && span.needzero != 0 { - delayedZeroing = true - } - if !noscan { - // Tell the GC not to look at this yet. - span.largeType = nil - header = &span.largeType - } + // Tiny allocator combines several tiny allocation requests + // into a single memory block. The resulting memory block + // is freed when all subobjects are unreachable. The subobjects + // must be noscan (don't have pointers), this ensures that + // the amount of potentially wasted memory is bounded. + // + // Size of the memory block used for combining (maxTinySize) is tunable. + // Current setting is 16 bytes, which relates to 2x worst case memory + // wastage (when all but one subobjects are unreachable). + // 8 bytes would result in no wastage at all, but provides less + // opportunities for combining. + // 32 bytes provides more opportunities for combining, + // but can lead to 4x worst case wastage. + // The best case winning is 8x regardless of block size. + // + // Objects obtained from tiny allocator must not be freed explicitly. + // So when an object will be freed explicitly, we ensure that + // its size >= maxTinySize. + // + // SetFinalizer has a special case for objects potentially coming + // from tiny allocator, it such case it allows to set finalizers + // for an inner byte of a memory block. + // + // The main targets of tiny allocator are small strings and + // standalone escaping variables. On a json benchmark + // the allocator reduces number of allocations by ~12% and + // reduces heap size by ~20%. + c := getMCache(mp) + off := c.tinyoffset + // Align tiny pointer for required (conservative) alignment. + if size&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && size == 12 { + // Conservatively align 12-byte objects to 8 bytes on 32-bit + // systems so that objects whose first field is a 64-bit + // value is aligned to 8 bytes and does not cause a fault on + // atomic access. See issue 37262. + // TODO(mknyszek): Remove this workaround if/when issue 36606 + // is resolved. + off = alignUp(off, 8) + } else if size&3 == 0 { + off = alignUp(off, 4) + } else if size&1 == 0 { + off = alignUp(off, 2) } - if !noscan && !delayedZeroing { - c.scanAlloc += heapSetType(uintptr(x), dataSize, typ, header, span) + if off+size <= maxTinySize && c.tiny != 0 { + // The object fits into existing tiny block. + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + size + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + return x, 0 + } + // Allocate a new maxTinySize block. + checkGCTrigger := false + span := c.alloc[tinySpanClass] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + // See if we need to replace the existing tiny block with the new one + // based on amount of remaining free space. + if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { + // Note: disabled when race detector is on, see comment near end of this function. + c.tiny = uintptr(x) + c.tinyoffset = size } // Ensure that the stores above that initialize x to @@ -1214,98 +1207,33 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // All slots hold nil so no scanning is needed. // This may be racing with GC so do it atomically if there can be // a race marking the bit. - if gcphase != _GCoff { + if writeBarrier.enabled { gcmarknewobject(span, uintptr(x)) } - if raceenabled { - racemalloc(x, size) - } - - if msanenabled { - msanmalloc(x, size) - } - - if asanenabled { - // We should only read/write the memory with the size asked by the user. - // The rest of the allocated memory should be poisoned, so that we can report - // errors when accessing poisoned memory. - // The allocated memory is larger than required userSize, it will also include - // redzone and some other padding bytes. - rzBeg := unsafe.Add(x, userSize) - asanpoison(rzBeg, size-userSize) - asanunpoison(x, userSize) - } - + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // // TODO(mknyszek): We should really count the header as part // of gc_sys or something. The code below just pretends it is // internal fragmentation and matches the GC's accounting by // using the whole allocation slot. - fullSize := span.elemsize - if rate := MemProfileRate; rate > 0 { - // Note cache c only valid while m acquired; see #47302 - // - // N.B. Use the full size because that matches how the GC - // will update the mem profile on the "free" side. - if rate != 1 && fullSize < c.nextSample { - c.nextSample -= fullSize - } else { - profilealloc(mp, x, fullSize) - } + c.nextSample -= int64(span.elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, span.elemsize) } mp.mallocing = 0 releasem(mp) - // Objects can be zeroed late in a context where preemption can occur. - // If the object contains pointers, its pointer data must be cleared - // or otherwise indicate that the GC shouldn't scan it. - // x will keep the memory alive. - if delayedZeroing { - // N.B. size == fullSize always in this case. - memclrNoHeapPointersChunked(size, x) // This is a possible preemption point: see #47302 - - // Finish storing the type information for this case. - if !noscan { - mp := acquirem() - getMCache(mp).scanAlloc += heapSetType(uintptr(x), dataSize, typ, header, span) - - // Publish the type information with the zeroed memory. - publicationBarrier() - releasem(mp) - } - } - - if debug.malloc { - if inittrace.active && inittrace.id == getg().goid { - // Init functions are executed sequentially in a single goroutine. - inittrace.bytes += uint64(fullSize) - } - - if traceAllocFreeEnabled() { - trace := traceAcquire() - if trace.ok() { - trace.HeapObjectAlloc(uintptr(x), typ) - traceRelease(trace) - } - } - } - - if assistG != nil { - // Account for internal fragmentation in the assist - // debt now that we know it. - // - // N.B. Use the full size because that's how the rest - // of the GC accounts for bytes marked. - assistG.gcAssistBytes -= int64(fullSize - dataSize) - } - - if shouldhelpgc { + if checkGCTrigger { if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { gcStart(t) } } - if raceenabled && noscan && dataSize < maxTinySize { + if raceenabled { // Pad tinysize allocations so they are aligned with the end // of the tinyalloc region. This ensures that any arithmetic // that goes off the top end of the object will be detectable @@ -1318,10 +1246,414 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // TODO: enable this padding for all allocations, not just // tinyalloc ones. It's tricky because of pointer maps. // Maybe just all noscan objects? - x = add(x, size-dataSize) + x = add(x, span.elemsize-size) + } + return x, span.elemsize +} + +func mallocgcSmallNoscan(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ != nil && typ.Pointers() { + throw("expected noscan type for noscan alloc") + } + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + var sizeclass uint8 + if size <= smallSizeMax-8 { + sizeclass = size_to_class8[divRoundUp(size, smallSizeDiv)] + } else { + sizeclass = size_to_class128[divRoundUp(size-smallSizeMax, largeSizeDiv)] + } + size = uintptr(class_to_size[sizeclass]) + spc := makeSpanClass(sizeclass, true) + span := c.alloc[spc] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, size) } - return x + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + // As x and the heap bits are initialized, update + // freeIndexForScan now so x is seen by the GC + // (including conservative scan) as an allocated object. + // While this pointer can't escape into user code as a + // _live_ pointer until we return, conservative scanning + // may find a dead pointer that happens to point into this + // object. Delaying this update until now ensures that + // conservative scanning considers this pointer dead until + // this point. + span.freeIndexForScan = span.freeindex + + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + if writeBarrier.enabled { + gcmarknewobject(span, uintptr(x)) + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + return x, size +} + +func mallocgcSmallScanNoHeader(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ == nil || !typ.Pointers() { + throw("noscan allocated in scan-only path") + } + if !heapBitsInSpan(size) { + throw("heap bits in not in span for non-header-only path") + } + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + sizeclass := size_to_class8[divRoundUp(size, smallSizeDiv)] + spc := makeSpanClass(sizeclass, false) + span := c.alloc[spc] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, size) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + // initHeapBits already set the pointer bits for the 8-byte sizeclass + // on 64-bit platforms. + c.scanAlloc += 8 + } else { + c.scanAlloc += heapSetTypeNoHeader(uintptr(x), size, typ, span) + } + size = uintptr(class_to_size[sizeclass]) + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + // As x and the heap bits are initialized, update + // freeIndexForScan now so x is seen by the GC + // (including conservative scan) as an allocated object. + // While this pointer can't escape into user code as a + // _live_ pointer until we return, conservative scanning + // may find a dead pointer that happens to point into this + // object. Delaying this update until now ensures that + // conservative scanning considers this pointer dead until + // this point. + span.freeIndexForScan = span.freeindex + + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + if writeBarrier.enabled { + gcmarknewobject(span, uintptr(x)) + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + return x, size +} + +func mallocgcSmallScanHeader(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ == nil || !typ.Pointers() { + throw("noscan allocated in scan-only path") + } + if heapBitsInSpan(size) { + throw("heap bits in span for header-only path") + } + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + size += mallocHeaderSize + var sizeclass uint8 + if size <= smallSizeMax-8 { + sizeclass = size_to_class8[divRoundUp(size, smallSizeDiv)] + } else { + sizeclass = size_to_class128[divRoundUp(size-smallSizeMax, largeSizeDiv)] + } + size = uintptr(class_to_size[sizeclass]) + spc := makeSpanClass(sizeclass, false) + span := c.alloc[spc] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, size) + } + header := (**_type)(x) + x = add(x, mallocHeaderSize) + c.scanAlloc += heapSetTypeSmallHeader(uintptr(x), size-mallocHeaderSize, typ, header, span) + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + // As x and the heap bits are initialized, update + // freeIndexForScan now so x is seen by the GC + // (including conservative scan) as an allocated object. + // While this pointer can't escape into user code as a + // _live_ pointer until we return, conservative scanning + // may find a dead pointer that happens to point into this + // object. Delaying this update until now ensures that + // conservative scanning considers this pointer dead until + // this point. + span.freeIndexForScan = span.freeindex + + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + if writeBarrier.enabled { + gcmarknewobject(span, uintptr(x)) + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + return x, size +} + +func mallocgcLarge(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + } + mp.mallocing = 1 + + c := getMCache(mp) + // For large allocations, keep track of zeroed state so that + // bulk zeroing can be happen later in a preemptible context. + span := c.allocLarge(size, typ == nil || !typ.Pointers()) + span.freeindex = 1 + span.allocCount = 1 + span.largeType = nil // Tell the GC not to look at this yet. + size = span.elemsize + x := unsafe.Pointer(span.base()) + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + // As x and the heap bits are initialized, update + // freeIndexForScan now so x is seen by the GC + // (including conservative scan) as an allocated object. + // While this pointer can't escape into user code as a + // _live_ pointer until we return, conservative scanning + // may find a dead pointer that happens to point into this + // object. Delaying this update until now ensures that + // conservative scanning considers this pointer dead until + // this point. + span.freeIndexForScan = span.freeindex + + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + if writeBarrier.enabled { + gcmarknewobject(span, uintptr(x)) + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) + } + mp.mallocing = 0 + releasem(mp) + + // Check to see if we need to trigger the GC. + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + + // Objects can be zeroed late in a context where preemption can occur. + // If the object contains pointers, its pointer data must be cleared + // or otherwise indicate that the GC shouldn't scan it. + // x will keep the memory alive. + if noscan := typ == nil || !typ.Pointers(); !noscan || (needzero && span.needzero != 0) { + // N.B. size == fullSize always in this case. + memclrNoHeapPointersChunked(size, x) // This is a possible preemption point: see #47302 + + // Finish storing the type information for this case. + mp := acquirem() + if !noscan { + getMCache(mp).scanAlloc += heapSetTypeLarge(uintptr(x), size, typ, span) + } + // Publish the object with the now-zeroed memory. + publicationBarrier() + releasem(mp) + } + return x, size +} + +func preMallocgcDebug(size uintptr, typ *_type) unsafe.Pointer { + if debug.sbrk != 0 { + align := uintptr(16) + if typ != nil { + // TODO(austin): This should be just + // align = uintptr(typ.align) + // but that's only 4 on 32-bit platforms, + // even if there's a uint64 field in typ (see #599). + // This causes 64-bit atomic accesses to panic. + // Hence, we use stricter alignment that matches + // the normal allocator better. + if size&7 == 0 { + align = 8 + } else if size&3 == 0 { + align = 4 + } else if size&1 == 0 { + align = 2 + } else { + align = 1 + } + } + return persistentalloc(size, align, &memstats.other_sys) + } + if inittrace.active && inittrace.id == getg().goid { + // Init functions are executed sequentially in a single goroutine. + inittrace.allocs += 1 + } + return nil +} + +func postMallocgcDebug(x unsafe.Pointer, elemsize uintptr, typ *_type) { + if inittrace.active && inittrace.id == getg().goid { + // Init functions are executed sequentially in a single goroutine. + inittrace.bytes += uint64(elemsize) + } + + if traceAllocFreeEnabled() { + trace := traceAcquire() + if trace.ok() { + trace.HeapObjectAlloc(uintptr(x), typ) + traceRelease(trace) + } + } } // deductAssistCredit reduces the current G's assist credit @@ -1330,26 +1662,22 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // Caller must be preemptible. // // Returns the G for which the assist credit was accounted. -func deductAssistCredit(size uintptr) *g { - var assistG *g - if gcBlackenEnabled != 0 { - // Charge the current user G for this allocation. - assistG = getg() - if assistG.m.curg != nil { - assistG = assistG.m.curg - } - // Charge the allocation against the G. We'll account - // for internal fragmentation at the end of mallocgc. - assistG.gcAssistBytes -= int64(size) - - if assistG.gcAssistBytes < 0 { - // This G is in debt. Assist the GC to correct - // this before allocating. This must happen - // before disabling preemption. - gcAssistAlloc(assistG) - } +func deductAssistCredit(size uintptr) { + // Charge the current user G for this allocation. + assistG := getg() + if assistG.m.curg != nil { + assistG = assistG.m.curg + } + // Charge the allocation against the G. We'll account + // for internal fragmentation at the end of mallocgc. + assistG.gcAssistBytes -= int64(size) + + if assistG.gcAssistBytes < 0 { + // This G is in debt. Assist the GC to correct + // this before allocating. This must happen + // before disabling preemption. + gcAssistAlloc(assistG) } - return assistG } // memclrNoHeapPointersChunked repeatedly calls memclrNoHeapPointers @@ -1386,6 +1714,11 @@ func newobject(typ *_type) unsafe.Pointer { return mallocgc(typ.Size_, typ, true) } +//go:linkname maps_newobject internal/runtime/maps.newobject +func maps_newobject(typ *_type) unsafe.Pointer { + return newobject(typ) +} + // reflect_unsafe_New is meant for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: @@ -1450,11 +1783,21 @@ func reflect_unsafe_NewArray(typ *_type, n int) unsafe.Pointer { return newarray(typ, n) } +//go:linkname maps_newarray internal/runtime/maps.newarray +func maps_newarray(typ *_type, n int) unsafe.Pointer { + return newarray(typ, n) +} + +// profilealloc resets the current mcache's nextSample counter and +// records a memory profile sample. +// +// The caller must be non-preemptible and have a P. func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { c := getMCache(mp) if c == nil { throw("profilealloc called without a P or outside bootstrapping") } + c.memProfRate = MemProfileRate c.nextSample = nextSample() mProf_Malloc(mp, x, size) } @@ -1466,12 +1809,13 @@ func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { // processes, the distance between two samples follows the exponential // distribution (exp(MemProfileRate)), so the best return value is a random // number taken from an exponential distribution whose mean is MemProfileRate. -func nextSample() uintptr { +func nextSample() int64 { + if MemProfileRate == 0 { + // Basically never sample. + return maxInt64 + } if MemProfileRate == 1 { - // Callers assign our return value to - // mcache.next_sample, but next_sample is not used - // when the rate is 1. So avoid the math below and - // just return something. + // Sample immediately. return 0 } if GOOS == "plan9" { @@ -1481,7 +1825,7 @@ func nextSample() uintptr { } } - return uintptr(fastexprand(MemProfileRate)) + return int64(fastexprand(MemProfileRate)) } // fastexprand returns a random number from an exponential distribution with @@ -1516,14 +1860,14 @@ func fastexprand(mean int) int32 { // nextSampleNoFP is similar to nextSample, but uses older, // simpler code to avoid floating point. -func nextSampleNoFP() uintptr { +func nextSampleNoFP() int64 { // Set first allocation sample size. rate := MemProfileRate if rate > 0x3fffffff { // make 2*rate not overflow rate = 0x3fffffff } if rate != 0 { - return uintptr(cheaprandn(uint32(2 * rate))) + return int64(cheaprandn(uint32(2 * rate))) } return 0 } @@ -1555,7 +1899,11 @@ var persistentChunks *notInHeap // sysStat must be non-nil. // // Consider marking persistentalloc'd types not in heap by embedding -// runtime/internal/sys.NotInHeap. +// internal/runtime/sys.NotInHeap. +// +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nosplit func persistentalloc(size, align uintptr, sysStat *sysMemStat) unsafe.Pointer { var p *notInHeap systemstack(func() { @@ -1697,7 +2045,7 @@ func (l *linearAlloc) alloc(size, align uintptr, sysStat *sysMemStat) unsafe.Poi // like sysAlloc or persistentAlloc. // // In general, it's better to use real types which embed -// runtime/internal/sys.NotInHeap, but this serves as a generic type +// internal/runtime/sys.NotInHeap, but this serves as a generic type // for situations where that isn't possible (like in the allocators). // // TODO: Use this as the return type of sysAlloc, persistentAlloc, etc? @@ -1707,9 +2055,9 @@ func (p *notInHeap) add(bytes uintptr) *notInHeap { return (*notInHeap)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + bytes)) } -// computeRZlog computes the size of the redzone. +// redZoneSize computes the size of the redzone for a given allocation. // Refer to the implementation of the compiler-rt. -func computeRZlog(userSize uintptr) uintptr { +func redZoneSize(userSize uintptr) uintptr { switch { case userSize <= (64 - 16): return 16 << 0 diff --git a/src/runtime/malloc_test.go b/src/runtime/malloc_test.go index 8c162fbe..67bceef2 100644 --- a/src/runtime/malloc_test.go +++ b/src/runtime/malloc_test.go @@ -7,6 +7,7 @@ package runtime_test import ( "flag" "fmt" + "internal/asan" "internal/race" "internal/testenv" "os" @@ -157,6 +158,9 @@ func TestTinyAlloc(t *testing.T) { if runtime.Raceenabled { t.Skip("tinyalloc suppressed when running in race mode") } + if asan.Enabled { + t.Skip("tinyalloc suppressed when running in asan mode due to redzone") + } const N = 16 var v [N]unsafe.Pointer for i := range v { @@ -182,6 +186,9 @@ func TestTinyAllocIssue37262(t *testing.T) { if runtime.Raceenabled { t.Skip("tinyalloc suppressed when running in race mode") } + if asan.Enabled { + t.Skip("tinyalloc suppressed when running in asan mode due to redzone") + } // Try to cause an alignment access fault // by atomically accessing the first 64-bit // value of a tiny-allocated object. diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go index 43d1accb..43c8f0bb 100644 --- a/src/runtime/map_benchmark_test.go +++ b/src/runtime/map_benchmark_test.go @@ -5,13 +5,20 @@ package runtime_test import ( + "encoding/binary" + "flag" "fmt" "math/rand" + "runtime" + "slices" "strconv" "strings" "testing" + "unsafe" ) +var mapbench = flag.Bool("mapbench", false, "enable the full set of map benchmark variants") + const size = 10 func BenchmarkHashStringSpeed(b *testing.B) { @@ -189,10 +196,12 @@ func BenchmarkSmallStrMap(b *testing.B) { } } -func BenchmarkMapStringKeysEight_16(b *testing.B) { benchmarkMapStringKeysEight(b, 16) } -func BenchmarkMapStringKeysEight_32(b *testing.B) { benchmarkMapStringKeysEight(b, 32) } -func BenchmarkMapStringKeysEight_64(b *testing.B) { benchmarkMapStringKeysEight(b, 64) } -func BenchmarkMapStringKeysEight_1M(b *testing.B) { benchmarkMapStringKeysEight(b, 1<<20) } +func BenchmarkMapStringKeysEight_16(b *testing.B) { benchmarkMapStringKeysEight(b, 16) } +func BenchmarkMapStringKeysEight_32(b *testing.B) { benchmarkMapStringKeysEight(b, 32) } +func BenchmarkMapStringKeysEight_64(b *testing.B) { benchmarkMapStringKeysEight(b, 64) } +func BenchmarkMapStringKeysEight_128(b *testing.B) { benchmarkMapStringKeysEight(b, 128) } +func BenchmarkMapStringKeysEight_256(b *testing.B) { benchmarkMapStringKeysEight(b, 256) } +func BenchmarkMapStringKeysEight_1M(b *testing.B) { benchmarkMapStringKeysEight(b, 1<<20) } func benchmarkMapStringKeysEight(b *testing.B, keySize int) { m := make(map[string]bool) @@ -206,17 +215,6 @@ func benchmarkMapStringKeysEight(b *testing.B, keySize int) { } } -func BenchmarkIntMap(b *testing.B) { - m := make(map[int]bool) - for i := 0; i < 8; i++ { - m[i] = true - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = m[7] - } -} - func BenchmarkMapFirst(b *testing.B) { for n := 1; n <= 16; n++ { b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { @@ -260,12 +258,41 @@ func BenchmarkMapLast(b *testing.B) { } } +func cyclicPermutation(n int) []int { + // From https://crypto.stackexchange.com/questions/51787/creating-single-cycle-permutations + p := rand.New(rand.NewSource(1)).Perm(n) + inc := make([]int, n) + pInv := make([]int, n) + for i := 0; i < n; i++ { + inc[i] = (i + 1) % n + pInv[p[i]] = i + } + res := make([]int, n) + for i := 0; i < n; i++ { + res[i] = pInv[inc[p[i]]] + } + + // Test result. + j := 0 + for i := 0; i < n-1; i++ { + j = res[j] + if j == 0 { + panic("got back to 0 too early") + } + } + j = res[j] + if j != 0 { + panic("didn't get back to 0") + } + return res +} + func BenchmarkMapCycle(b *testing.B) { // Arrange map entries to be a permutation, so that // we hit all entries, and one lookup is data dependent // on the previous lookup. const N = 3127 - p := rand.New(rand.NewSource(1)).Perm(N) + p := cyclicPermutation(N) m := map[int]int{} for i := 0; i < N; i++ { m[i] = p[i] @@ -333,27 +360,6 @@ func BenchmarkNewSmallMap(b *testing.B) { } } -func BenchmarkMapIter(b *testing.B) { - m := make(map[int]bool) - for i := 0; i < 8; i++ { - m[i] = true - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - for range m { - } - } -} - -func BenchmarkMapIterEmpty(b *testing.B) { - m := make(map[int]bool) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for range m { - } - } -} - func BenchmarkSameLengthMap(b *testing.B) { // long strings, same length, differ in first few // and last few bytes. @@ -368,28 +374,6 @@ func BenchmarkSameLengthMap(b *testing.B) { } } -type BigKey [3]int64 - -func BenchmarkBigKeyMap(b *testing.B) { - m := make(map[BigKey]bool) - k := BigKey{3, 4, 5} - m[k] = true - for i := 0; i < b.N; i++ { - _ = m[k] - } -} - -type BigVal [3]int64 - -func BenchmarkBigValMap(b *testing.B) { - m := make(map[BigKey]BigVal) - k := BigKey{3, 4, 5} - m[k] = BigVal{6, 7, 8} - for i := 0; i < b.N; i++ { - _ = m[k] - } -} - func BenchmarkSmallKeyMap(b *testing.B) { m := make(map[int16]bool) m[5] = true @@ -538,3 +522,669 @@ func BenchmarkNewEmptyMapHintGreaterThan8(b *testing.B) { _ = make(map[int]int, hintGreaterThan8) } } + +func benchSizes(f func(b *testing.B, n int)) func(*testing.B) { + var cases = []int{ + 0, + 6, + 12, + 18, + 24, + 30, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + 1 << 16, + 1 << 18, + 1 << 20, + 1 << 22, + } + + // Cases enabled by default. Set -mapbench for the remainder. + // + // With the other type combinations, there are literally thousands of + // variations. It take too long to run all of these as part of + // builders. + byDefault := map[int]bool{ + 6: true, + 64: true, + 1 << 16: true, + } + + return func(b *testing.B) { + for _, n := range cases { + b.Run("len="+strconv.Itoa(n), func(b *testing.B) { + if !*mapbench && !byDefault[n] { + b.Skip("Skipped because -mapbench=false") + } + + f(b, n) + }) + } + } +} +func smallBenchSizes(f func(b *testing.B, n int)) func(*testing.B) { + return func(b *testing.B) { + for n := 1; n <= 8; n++ { + b.Run("len="+strconv.Itoa(n), func(b *testing.B) { + f(b, n) + }) + } + } +} + +// A 16 byte type. +type smallType [16]byte + +// A 512 byte type. +type mediumType [1 << 9]byte + +// A 4KiB type. +type bigType [1 << 12]byte + +type mapBenchmarkKeyType interface { + int32 | int64 | string | smallType | mediumType | bigType | *int32 +} + +type mapBenchmarkElemType interface { + mapBenchmarkKeyType | []int32 +} + +func genIntValues[T int | int32 | int64](start, end int) []T { + vals := make([]T, 0, end-start) + for i := start; i < end; i++ { + vals = append(vals, T(i)) + } + return vals +} + +func genStringValues(start, end int) []string { + vals := make([]string, 0, end-start) + for i := start; i < end; i++ { + vals = append(vals, strconv.Itoa(i)) + } + return vals +} + +func genSmallValues(start, end int) []smallType { + vals := make([]smallType, 0, end-start) + for i := start; i < end; i++ { + var v smallType + binary.NativeEndian.PutUint64(v[:], uint64(i)) + vals = append(vals, v) + } + return vals +} + +func genMediumValues(start, end int) []mediumType { + vals := make([]mediumType, 0, end-start) + for i := start; i < end; i++ { + var v mediumType + binary.NativeEndian.PutUint64(v[:], uint64(i)) + vals = append(vals, v) + } + return vals +} + +func genBigValues(start, end int) []bigType { + vals := make([]bigType, 0, end-start) + for i := start; i < end; i++ { + var v bigType + binary.NativeEndian.PutUint64(v[:], uint64(i)) + vals = append(vals, v) + } + return vals +} + +func genPtrValues[T any](start, end int) []*T { + // Start and end don't mean much. Each pointer by definition has a + // unique identity. + vals := make([]*T, 0, end-start) + for i := start; i < end; i++ { + v := new(T) + vals = append(vals, v) + } + return vals +} + +func genIntSliceValues[T int | int32 | int64](start, end int) [][]T { + vals := make([][]T, 0, end-start) + for i := start; i < end; i++ { + vals = append(vals, []T{T(i)}) + } + return vals +} + +func genValues[T mapBenchmarkElemType](start, end int) []T { + var t T + switch any(t).(type) { + case int32: + return any(genIntValues[int32](start, end)).([]T) + case int64: + return any(genIntValues[int64](start, end)).([]T) + case string: + return any(genStringValues(start, end)).([]T) + case smallType: + return any(genSmallValues(start, end)).([]T) + case mediumType: + return any(genMediumValues(start, end)).([]T) + case bigType: + return any(genBigValues(start, end)).([]T) + case *int32: + return any(genPtrValues[int32](start, end)).([]T) + case []int32: + return any(genIntSliceValues[int32](start, end)).([]T) + default: + panic("unreachable") + } +} + +// Avoid inlining to force a heap allocation. +// +//go:noinline +func newSink[T mapBenchmarkElemType]() *T { + return new(T) +} + +// Return a new maps filled with keys and elems. Both slices must be the same length. +func fillMap[K mapBenchmarkKeyType, E mapBenchmarkElemType](keys []K, elems []E) map[K]E { + m := make(map[K]E, len(keys)) + for i := range keys { + m[keys[i]] = elems[i] + } + return m +} + +func iterCount(b *testing.B, n int) int { + // Divide b.N by n so that the ns/op reports time per element, + // not time per full map iteration. This makes benchmarks of + // different map sizes more comparable. + // + // If size is zero we still need to do iterations. + if n == 0 { + return b.N + } + return b.N / n +} + +func checkAllocSize[K, E any](b *testing.B, n int) { + var k K + size := uint64(n) * uint64(unsafe.Sizeof(k)) + var e E + size += uint64(n) * uint64(unsafe.Sizeof(e)) + + if size >= 1<<30 { + b.Skipf("Total key+elem size %d exceeds 1GiB", size) + } +} + +func benchmarkMapIter[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + iterations := iterCount(b, n) + sinkK := newSink[K]() + sinkE := newSink[E]() + b.ResetTimer() + + for i := 0; i < iterations; i++ { + for k, e := range m { + *sinkK = k + *sinkE = e + } + } +} + +func BenchmarkMapIter(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapIter[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapIter[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapIter[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapIter[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapIter[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapIter[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapIter[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapIter[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapIter[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapIter[int32, *int32])) +} + +func benchmarkMapIterLowLoad[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + // Only insert one entry regardless of map size. + k := genValues[K](0, 1) + e := genValues[E](0, 1) + + m := make(map[K]E, n) + for i := range k { + m[k[i]] = e[i] + } + + iterations := iterCount(b, n) + sinkK := newSink[K]() + sinkE := newSink[E]() + b.ResetTimer() + + for i := 0; i < iterations; i++ { + for k, e := range m { + *sinkK = k + *sinkE = e + } + } +} + +func BenchmarkMapIterLowLoad(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapIterLowLoad[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapIterLowLoad[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapIterLowLoad[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapIterLowLoad[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapIterLowLoad[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapIterLowLoad[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapIterLowLoad[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapIterLowLoad[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapIterLowLoad[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapIterLowLoad[int32, *int32])) +} + +func benchmarkMapAccessHit[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't access empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + sink := newSink[E]() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + *sink = m[k[i%n]] + } +} + +func BenchmarkMapAccessHit(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAccessHit[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAccessHit[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAccessHit[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAccessHit[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAccessHit[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAccessHit[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAccessHit[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAccessHit[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAccessHit[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAccessHit[int32, *int32])) +} + +var sinkOK bool + +func benchmarkMapAccessMiss[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + if n == 0 { // Create a lookup values for empty maps. + n = 1 + } + w := genValues[K](n, 2*n) + b.ResetTimer() + + var ok bool + for i := 0; i < b.N; i++ { + _, ok = m[w[i%n]] + } + + sinkOK = ok +} + +func BenchmarkMapAccessMiss(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAccessMiss[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAccessMiss[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAccessMiss[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAccessMiss[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAccessMiss[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAccessMiss[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAccessMiss[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAccessMiss[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAccessMiss[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAccessMiss[int32, *int32])) +} + +// Assign to a key that already exists. +func benchmarkMapAssignExists[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't assign to existing keys in empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignExists(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignExists[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignExists[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignExists[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignExists[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignExists[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignExists[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignExists[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignExists[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignExists[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignExists[int32, *int32])) +} + +// Fill a map of size n with no hint. Time is per-key. A new map is created +// every n assignments. +// +// TODO(prattmic): Results don't make much sense if b.N < n. +// TODO(prattmic): Measure distribution of assign time to reveal the grow +// latency. +func benchmarkMapAssignFillNoHint[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + b.ResetTimer() + + var m map[K]E + for i := 0; i < b.N; i++ { + if i%n == 0 { + m = make(map[K]E) + } + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignFillNoHint(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignFillNoHint[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignFillNoHint[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignFillNoHint[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignFillNoHint[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignFillNoHint[int32, *int32])) +} + +// Identical to benchmarkMapAssignFillNoHint, but additionally measures the +// latency of each mapassign to report tail latency due to map grow. +func benchmarkMapAssignGrowLatency[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + + // Store the run time of each mapassign. Keeping the full data rather + // than a histogram provides higher precision. b.N tends to be <10M, so + // the memory requirement isn't too bad. + sample := make([]int64, b.N) + + b.ResetTimer() + + var m map[K]E + for i := 0; i < b.N; i++ { + if i%n == 0 { + m = make(map[K]E) + } + start := runtime.Nanotime() + m[k[i%n]] = e[i%n] + end := runtime.Nanotime() + sample[i] = end - start + } + + b.StopTimer() + + slices.Sort(sample) + // TODO(prattmic): Grow is so rare that even p99.99 often doesn't + // display a grow case. Switch to a more direct measure of grow cases + // only? + b.ReportMetric(float64(sample[int(float64(len(sample))*0.5)]), "p50-ns/op") + b.ReportMetric(float64(sample[int(float64(len(sample))*0.99)]), "p99-ns/op") + b.ReportMetric(float64(sample[int(float64(len(sample))*0.999)]), "p99.9-ns/op") + b.ReportMetric(float64(sample[int(float64(len(sample))*0.9999)]), "p99.99-ns/op") + b.ReportMetric(float64(sample[len(sample)-1]), "p100-ns/op") +} + +func BenchmarkMapAssignGrowLatency(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignGrowLatency[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignGrowLatency[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignGrowLatency[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignGrowLatency[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignGrowLatency[int32, *int32])) +} + +// Fill a map of size n with size hint. Time is per-key. A new map is created +// every n assignments. +// +// TODO(prattmic): Results don't make much sense if b.N < n. +func benchmarkMapAssignFillHint[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + b.ResetTimer() + + var m map[K]E + for i := 0; i < b.N; i++ { + if i%n == 0 { + m = make(map[K]E, n) + } + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignFillHint(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignFillHint[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignFillHint[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignFillHint[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignFillHint[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignFillHint[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignFillHint[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignFillHint[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignFillHint[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignFillHint[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignFillHint[int32, *int32])) +} + +// Fill a map of size n, reusing the same map. Time is per-key. The map is +// cleared every n assignments. +// +// TODO(prattmic): Results don't make much sense if b.N < n. +func benchmarkMapAssignFillClear[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if i%n == 0 { + clear(m) + } + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignFillClear(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignFillClear[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignFillClear[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignFillClear[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignFillClear[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignFillClear[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignFillClear[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignFillClear[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignFillClear[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignFillClear[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignFillClear[int32, *int32])) +} + +// Modify values using +=. +func benchmarkMapAssignAddition[K mapBenchmarkKeyType, E int32 | int64 | string](b *testing.B, n int) { + if n == 0 { + b.Skip("can't modify empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + m[k[i%n]] += e[i%n] + } +} + +func BenchmarkMapAssignAddition(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignAddition[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignAddition[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignAddition[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignAddition[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignAddition[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignAddition[bigType, int32])) +} + +// Modify values append. +func benchmarkMapAssignAppend[K mapBenchmarkKeyType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't modify empty map via append") + } + checkAllocSize[K, []int32](b, n) + k := genValues[K](0, n) + e := genValues[[]int32](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + m[k[i%n]] = append(m[k[i%n]], e[i%n][0]) + } +} + +func BenchmarkMapAssignAppend(b *testing.B) { + b.Run("Key=int32/Elem=[]int32", benchSizes(benchmarkMapAssignAppend[int32])) + b.Run("Key=int64/Elem=[]int32", benchSizes(benchmarkMapAssignAppend[int64])) + b.Run("Key=string/Elem=[]int32", benchSizes(benchmarkMapAssignAppend[string])) +} + +func benchmarkMapDelete[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't delete from empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if len(m) == 0 { + // We'd like to StopTimer while refilling the map, but + // it is way too expensive and thus makes the benchmark + // take a long time. See https://go.dev/issue/20875. + for j := range k { + m[k[j]] = e[j] + } + } + delete(m, k[i%n]) + } +} + +func BenchmarkMapDelete(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapDelete[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapDelete[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapDelete[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapDelete[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapDelete[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapDelete[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapDelete[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapDelete[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapDelete[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapDelete[int32, *int32])) +} + +// Use iterator to pop an element. We want this to be fast, see +// https://go.dev/issue/8412. +func benchmarkMapPop[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't delete from empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if len(m) == 0 { + // We'd like to StopTimer while refilling the map, but + // it is way too expensive and thus makes the benchmark + // take a long time. See https://go.dev/issue/20875. + for j := range k { + m[k[j]] = e[j] + } + } + for key := range m { + delete(m, key) + break + } + } +} + +func BenchmarkMapPop(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapPop[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapPop[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapPop[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapPop[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapPop[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapPop[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapPop[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapPop[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapPop[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapPop[int32, *int32])) +} + +func BenchmarkMapDeleteLargeKey(b *testing.B) { + m := map[string]int{} + for i := range 9 { + m[fmt.Sprintf("%d", i)] = i + } + key := strings.Repeat("*", 10000) + for range b.N { + delete(m, key) + } +} + +func BenchmarkMapSmallAccessHit(b *testing.B) { + b.Run("Key=int32/Elem=int32", smallBenchSizes(benchmarkMapAccessHit[int32, int32])) + b.Run("Key=int64/Elem=int64", smallBenchSizes(benchmarkMapAccessHit[int64, int64])) + b.Run("Key=string/Elem=string", smallBenchSizes(benchmarkMapAccessHit[string, string])) +} +func BenchmarkMapSmallAccessMiss(b *testing.B) { + b.Run("Key=int32/Elem=int32", smallBenchSizes(benchmarkMapAccessMiss[int32, int32])) + b.Run("Key=int64/Elem=int64", smallBenchSizes(benchmarkMapAccessMiss[int64, int64])) + b.Run("Key=string/Elem=string", smallBenchSizes(benchmarkMapAccessMiss[string, string])) +} diff --git a/src/runtime/map_fast32.go b/src/runtime/map_fast32_noswiss.go similarity index 87% rename from src/runtime/map_fast32.go rename to src/runtime/map_fast32_noswiss.go index 0eb8562f..751717b6 100644 --- a/src/runtime/map_fast32.go +++ b/src/runtime/map_fast32_noswiss.go @@ -2,17 +2,20 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.swissmap + package runtime import ( "internal/abi" "internal/goarch" + "internal/runtime/sys" "unsafe" ) func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess1_fast32)) } if h == nil || h.count == 0 { @@ -41,9 +44,9 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { } } for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 4) { + for i, k := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, k = i+1, add(k, 4) { if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*4+i*uintptr(t.ValueSize)) + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*4+i*uintptr(t.ValueSize)) } } } @@ -61,7 +64,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { //go:linkname mapaccess2_fast32 func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess2_fast32)) } if h == nil || h.count == 0 { @@ -90,9 +93,9 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) { } } for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 4) { + for i, k := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, k = i+1, add(k, 4) { if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*4+i*uintptr(t.ValueSize)), true + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*4+i*uintptr(t.ValueSize)), true } } } @@ -103,7 +106,6 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) { // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. @@ -115,7 +117,7 @@ func mapassign_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { panic(plainError("assignment to entry in nil map")) } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast32)) } if h.flags&hashWriting != 0 { @@ -143,7 +145,7 @@ again: bucketloop: for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { if isEmpty(b.tophash[i]) { if insertb == nil { inserti = i @@ -183,7 +185,7 @@ bucketloop: insertb = h.newoverflow(t, b) inserti = 0 // not necessary, but avoids needlessly spilling inserti } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks + insertb.tophash[inserti&(abi.OldMapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*4) // store new key at insert position @@ -192,7 +194,7 @@ bucketloop: h.count++ done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*4+inserti*uintptr(t.ValueSize)) + elem := add(unsafe.Pointer(insertb), dataOffset+abi.OldMapBucketCount*4+inserti*uintptr(t.ValueSize)) if h.flags&hashWriting == 0 { fatal("concurrent map writes") } @@ -214,7 +216,7 @@ func mapassign_fast32ptr(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer panic(plainError("assignment to entry in nil map")) } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast32)) } if h.flags&hashWriting != 0 { @@ -242,7 +244,7 @@ again: bucketloop: for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { if isEmpty(b.tophash[i]) { if insertb == nil { inserti = i @@ -282,7 +284,7 @@ bucketloop: insertb = h.newoverflow(t, b) inserti = 0 // not necessary, but avoids needlessly spilling inserti } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks + insertb.tophash[inserti&(abi.OldMapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*4) // store new key at insert position @@ -291,7 +293,7 @@ bucketloop: h.count++ done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*4+inserti*uintptr(t.ValueSize)) + elem := add(unsafe.Pointer(insertb), dataOffset+abi.OldMapBucketCount*4+inserti*uintptr(t.ValueSize)) if h.flags&hashWriting == 0 { fatal("concurrent map writes") } @@ -301,7 +303,7 @@ done: func mapdelete_fast32(t *maptype, h *hmap, key uint32) { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapdelete_fast32)) } if h == nil || h.count == 0 { @@ -324,7 +326,7 @@ func mapdelete_fast32(t *maptype, h *hmap, key uint32) { bOrig := b search: for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 4) { + for i, k := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, k = i+1, add(k, 4) { if key != *(*uint32)(k) || isEmpty(b.tophash[i]) { continue } @@ -336,7 +338,7 @@ search: // 32 bits wide and the key is 32 bits wide also. *(*unsafe.Pointer)(k) = nil } - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*4+i*uintptr(t.ValueSize)) + e := add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*4+i*uintptr(t.ValueSize)) if t.Elem.Pointers() { memclrHasPointers(e, t.Elem.Size_) } else { @@ -345,7 +347,7 @@ search: b.tophash[i] = emptyOne // If the bucket now ends in a bunch of emptyOne states, // change those to emptyRest states. - if i == abi.MapBucketCount-1 { + if i == abi.OldMapBucketCount-1 { if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest { goto notLast } @@ -364,7 +366,7 @@ search: c := b for b = bOrig; b.overflow(t) != c; b = b.overflow(t) { } - i = abi.MapBucketCount - 1 + i = abi.OldMapBucketCount - 1 } else { i-- } @@ -412,7 +414,7 @@ func evacuate_fast32(t *maptype, h *hmap, oldbucket uintptr) { x := &xy[0] x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.BucketSize))) x.k = add(unsafe.Pointer(x.b), dataOffset) - x.e = add(x.k, abi.MapBucketCount*4) + x.e = add(x.k, abi.OldMapBucketCount*4) if !h.sameSizeGrow() { // Only calculate y pointers if we're growing bigger. @@ -420,13 +422,13 @@ func evacuate_fast32(t *maptype, h *hmap, oldbucket uintptr) { y := &xy[1] y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.BucketSize))) y.k = add(unsafe.Pointer(y.b), dataOffset) - y.e = add(y.k, abi.MapBucketCount*4) + y.e = add(y.k, abi.OldMapBucketCount*4) } for ; b != nil; b = b.overflow(t) { k := add(unsafe.Pointer(b), dataOffset) - e := add(k, abi.MapBucketCount*4) - for i := 0; i < abi.MapBucketCount; i, k, e = i+1, add(k, 4), add(e, uintptr(t.ValueSize)) { + e := add(k, abi.OldMapBucketCount*4) + for i := 0; i < abi.OldMapBucketCount; i, k, e = i+1, add(k, 4), add(e, uintptr(t.ValueSize)) { top := b.tophash[i] if isEmpty(top) { b.tophash[i] = evacuatedEmpty @@ -448,13 +450,13 @@ func evacuate_fast32(t *maptype, h *hmap, oldbucket uintptr) { b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY, enforced in makemap dst := &xy[useY] // evacuation destination - if dst.i == abi.MapBucketCount { + if dst.i == abi.OldMapBucketCount { dst.b = h.newoverflow(t, dst.b) dst.i = 0 dst.k = add(unsafe.Pointer(dst.b), dataOffset) - dst.e = add(dst.k, abi.MapBucketCount*4) + dst.e = add(dst.k, abi.OldMapBucketCount*4) } - dst.b.tophash[dst.i&(abi.MapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check + dst.b.tophash[dst.i&(abi.OldMapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check // Copy key. if goarch.PtrSize == 4 && t.Key.Pointers() && writeBarrier.enabled { diff --git a/src/runtime/map_fast32_swiss.go b/src/runtime/map_fast32_swiss.go new file mode 100644 index 00000000..0a241d37 --- /dev/null +++ b/src/runtime/map_fast32_swiss.go @@ -0,0 +1,55 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package runtime + +import ( + "internal/abi" + "internal/runtime/maps" + "unsafe" +) + +// Functions below pushed from internal/runtime/maps. + +//go:linkname mapaccess1_fast32 +func mapaccess1_fast32(t *abi.SwissMapType, m *maps.Map, key uint32) unsafe.Pointer + +// mapaccess2_fast32 should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapaccess2_fast32 +func mapaccess2_fast32(t *abi.SwissMapType, m *maps.Map, key uint32) (unsafe.Pointer, bool) + +// mapassign_fast32 should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapassign_fast32 +func mapassign_fast32(t *abi.SwissMapType, m *maps.Map, key uint32) unsafe.Pointer + +// mapassign_fast32ptr should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapassign_fast32ptr +func mapassign_fast32ptr(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer + +//go:linkname mapdelete_fast32 +func mapdelete_fast32(t *abi.SwissMapType, m *maps.Map, key uint32) diff --git a/src/runtime/map_fast64.go b/src/runtime/map_fast64_noswiss.go similarity index 87% rename from src/runtime/map_fast64.go rename to src/runtime/map_fast64_noswiss.go index aca60eb2..abb272d2 100644 --- a/src/runtime/map_fast64.go +++ b/src/runtime/map_fast64_noswiss.go @@ -2,17 +2,20 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.swissmap + package runtime import ( "internal/abi" "internal/goarch" + "internal/runtime/sys" "unsafe" ) func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess1_fast64)) } if h == nil || h.count == 0 { @@ -41,9 +44,9 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { } } for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 8) { + for i, k := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, k = i+1, add(k, 8) { if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*8+i*uintptr(t.ValueSize)) + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*8+i*uintptr(t.ValueSize)) } } } @@ -61,7 +64,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { //go:linkname mapaccess2_fast64 func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess2_fast64)) } if h == nil || h.count == 0 { @@ -90,9 +93,9 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) { } } for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 8) { + for i, k := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, k = i+1, add(k, 8) { if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*8+i*uintptr(t.ValueSize)), true + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*8+i*uintptr(t.ValueSize)), true } } } @@ -103,7 +106,6 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) { // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. @@ -115,7 +117,7 @@ func mapassign_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { panic(plainError("assignment to entry in nil map")) } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast64)) } if h.flags&hashWriting != 0 { @@ -143,7 +145,7 @@ again: bucketloop: for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { if isEmpty(b.tophash[i]) { if insertb == nil { insertb = b @@ -183,7 +185,7 @@ bucketloop: insertb = h.newoverflow(t, b) inserti = 0 // not necessary, but avoids needlessly spilling inserti } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks + insertb.tophash[inserti&(abi.OldMapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*8) // store new key at insert position @@ -192,7 +194,7 @@ bucketloop: h.count++ done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*8+inserti*uintptr(t.ValueSize)) + elem := add(unsafe.Pointer(insertb), dataOffset+abi.OldMapBucketCount*8+inserti*uintptr(t.ValueSize)) if h.flags&hashWriting == 0 { fatal("concurrent map writes") } @@ -204,7 +206,6 @@ done: // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. @@ -216,7 +217,7 @@ func mapassign_fast64ptr(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer panic(plainError("assignment to entry in nil map")) } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast64)) } if h.flags&hashWriting != 0 { @@ -244,7 +245,7 @@ again: bucketloop: for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { if isEmpty(b.tophash[i]) { if insertb == nil { insertb = b @@ -284,7 +285,7 @@ bucketloop: insertb = h.newoverflow(t, b) inserti = 0 // not necessary, but avoids needlessly spilling inserti } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks + insertb.tophash[inserti&(abi.OldMapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*8) // store new key at insert position @@ -293,7 +294,7 @@ bucketloop: h.count++ done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*8+inserti*uintptr(t.ValueSize)) + elem := add(unsafe.Pointer(insertb), dataOffset+abi.OldMapBucketCount*8+inserti*uintptr(t.ValueSize)) if h.flags&hashWriting == 0 { fatal("concurrent map writes") } @@ -303,7 +304,7 @@ done: func mapdelete_fast64(t *maptype, h *hmap, key uint64) { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapdelete_fast64)) } if h == nil || h.count == 0 { @@ -326,7 +327,7 @@ func mapdelete_fast64(t *maptype, h *hmap, key uint64) { bOrig := b search: for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 8) { + for i, k := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, k = i+1, add(k, 8) { if key != *(*uint64)(k) || isEmpty(b.tophash[i]) { continue } @@ -340,7 +341,7 @@ search: memclrHasPointers(k, 8) } } - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*8+i*uintptr(t.ValueSize)) + e := add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*8+i*uintptr(t.ValueSize)) if t.Elem.Pointers() { memclrHasPointers(e, t.Elem.Size_) } else { @@ -349,7 +350,7 @@ search: b.tophash[i] = emptyOne // If the bucket now ends in a bunch of emptyOne states, // change those to emptyRest states. - if i == abi.MapBucketCount-1 { + if i == abi.OldMapBucketCount-1 { if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest { goto notLast } @@ -368,7 +369,7 @@ search: c := b for b = bOrig; b.overflow(t) != c; b = b.overflow(t) { } - i = abi.MapBucketCount - 1 + i = abi.OldMapBucketCount - 1 } else { i-- } @@ -416,7 +417,7 @@ func evacuate_fast64(t *maptype, h *hmap, oldbucket uintptr) { x := &xy[0] x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.BucketSize))) x.k = add(unsafe.Pointer(x.b), dataOffset) - x.e = add(x.k, abi.MapBucketCount*8) + x.e = add(x.k, abi.OldMapBucketCount*8) if !h.sameSizeGrow() { // Only calculate y pointers if we're growing bigger. @@ -424,13 +425,13 @@ func evacuate_fast64(t *maptype, h *hmap, oldbucket uintptr) { y := &xy[1] y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.BucketSize))) y.k = add(unsafe.Pointer(y.b), dataOffset) - y.e = add(y.k, abi.MapBucketCount*8) + y.e = add(y.k, abi.OldMapBucketCount*8) } for ; b != nil; b = b.overflow(t) { k := add(unsafe.Pointer(b), dataOffset) - e := add(k, abi.MapBucketCount*8) - for i := 0; i < abi.MapBucketCount; i, k, e = i+1, add(k, 8), add(e, uintptr(t.ValueSize)) { + e := add(k, abi.OldMapBucketCount*8) + for i := 0; i < abi.OldMapBucketCount; i, k, e = i+1, add(k, 8), add(e, uintptr(t.ValueSize)) { top := b.tophash[i] if isEmpty(top) { b.tophash[i] = evacuatedEmpty @@ -452,13 +453,13 @@ func evacuate_fast64(t *maptype, h *hmap, oldbucket uintptr) { b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY, enforced in makemap dst := &xy[useY] // evacuation destination - if dst.i == abi.MapBucketCount { + if dst.i == abi.OldMapBucketCount { dst.b = h.newoverflow(t, dst.b) dst.i = 0 dst.k = add(unsafe.Pointer(dst.b), dataOffset) - dst.e = add(dst.k, abi.MapBucketCount*8) + dst.e = add(dst.k, abi.OldMapBucketCount*8) } - dst.b.tophash[dst.i&(abi.MapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check + dst.b.tophash[dst.i&(abi.OldMapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check // Copy key. if t.Key.Pointers() && writeBarrier.enabled { diff --git a/src/runtime/map_fast64_swiss.go b/src/runtime/map_fast64_swiss.go new file mode 100644 index 00000000..8b7fcf88 --- /dev/null +++ b/src/runtime/map_fast64_swiss.go @@ -0,0 +1,56 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package runtime + +import ( + "internal/abi" + "internal/runtime/maps" + "unsafe" +) + +// Functions below pushed from internal/runtime/maps. + +//go:linkname mapaccess1_fast64 +func mapaccess1_fast64(t *abi.SwissMapType, m *maps.Map, key uint64) unsafe.Pointer + +// mapaccess2_fast64 should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapaccess2_fast64 +func mapaccess2_fast64(t *abi.SwissMapType, m *maps.Map, key uint64) (unsafe.Pointer, bool) + +// mapassign_fast64 should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapassign_fast64 +func mapassign_fast64(t *abi.SwissMapType, m *maps.Map, key uint64) unsafe.Pointer + +// mapassign_fast64ptr should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapassign_fast64ptr +func mapassign_fast64ptr(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer + +//go:linkname mapdelete_fast64 +func mapdelete_fast64(t *abi.SwissMapType, m *maps.Map, key uint64) diff --git a/src/runtime/map_faststr.go b/src/runtime/map_faststr_noswiss.go similarity index 80% rename from src/runtime/map_faststr.go rename to src/runtime/map_faststr_noswiss.go index 5461a9f8..e8b6a3f1 100644 --- a/src/runtime/map_faststr.go +++ b/src/runtime/map_faststr_noswiss.go @@ -2,17 +2,20 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.swissmap + package runtime import ( "internal/abi" "internal/goarch" + "internal/runtime/sys" "unsafe" ) func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess1_faststr)) } if h == nil || h.count == 0 { @@ -27,7 +30,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { b := (*bmap)(h.buckets) if key.len < 32 { // short key, doing lots of comparisons is ok - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { + for i, kptr := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { k := (*stringStruct)(kptr) if k.len != key.len || isEmpty(b.tophash[i]) { if b.tophash[i] == emptyRest { @@ -36,14 +39,14 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { continue } if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) } } return unsafe.Pointer(&zeroVal[0]) } // long key, try not to do more comparisons than necessary - keymaybe := uintptr(abi.MapBucketCount) - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { + keymaybe := uintptr(abi.OldMapBucketCount) + for i, kptr := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { k := (*stringStruct)(kptr) if k.len != key.len || isEmpty(b.tophash[i]) { if b.tophash[i] == emptyRest { @@ -52,7 +55,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { continue } if k.str == key.str { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) } // check first 4 bytes if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) { @@ -62,16 +65,16 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) { continue } - if keymaybe != abi.MapBucketCount { + if keymaybe != abi.OldMapBucketCount { // Two keys are potential matches. Use hash to distinguish them. goto dohash } keymaybe = i } - if keymaybe != abi.MapBucketCount { + if keymaybe != abi.OldMapBucketCount { k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*goarch.PtrSize)) if memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+keymaybe*uintptr(t.ValueSize)) + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+keymaybe*uintptr(t.ValueSize)) } } return unsafe.Pointer(&zeroVal[0]) @@ -92,13 +95,13 @@ dohash: } top := tophash(hash) for ; b != nil; b = b.overflow(t) { - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { + for i, kptr := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { k := (*stringStruct)(kptr) if k.len != key.len || b.tophash[i] != top { continue } if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) } } } @@ -116,7 +119,7 @@ dohash: //go:linkname mapaccess2_faststr func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess2_faststr)) } if h == nil || h.count == 0 { @@ -131,7 +134,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { b := (*bmap)(h.buckets) if key.len < 32 { // short key, doing lots of comparisons is ok - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { + for i, kptr := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { k := (*stringStruct)(kptr) if k.len != key.len || isEmpty(b.tophash[i]) { if b.tophash[i] == emptyRest { @@ -140,14 +143,14 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { continue } if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true } } return unsafe.Pointer(&zeroVal[0]), false } // long key, try not to do more comparisons than necessary - keymaybe := uintptr(abi.MapBucketCount) - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { + keymaybe := uintptr(abi.OldMapBucketCount) + for i, kptr := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { k := (*stringStruct)(kptr) if k.len != key.len || isEmpty(b.tophash[i]) { if b.tophash[i] == emptyRest { @@ -156,7 +159,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { continue } if k.str == key.str { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true } // check first 4 bytes if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) { @@ -166,16 +169,16 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) { continue } - if keymaybe != abi.MapBucketCount { + if keymaybe != abi.OldMapBucketCount { // Two keys are potential matches. Use hash to distinguish them. goto dohash } keymaybe = i } - if keymaybe != abi.MapBucketCount { + if keymaybe != abi.OldMapBucketCount { k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*goarch.PtrSize)) if memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+keymaybe*uintptr(t.ValueSize)), true + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+keymaybe*uintptr(t.ValueSize)), true } } return unsafe.Pointer(&zeroVal[0]), false @@ -196,13 +199,13 @@ dohash: } top := tophash(hash) for ; b != nil; b = b.overflow(t) { - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { + for i, kptr := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { k := (*stringStruct)(kptr) if k.len != key.len || b.tophash[i] != top { continue } if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true + return add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true } } } @@ -213,7 +216,6 @@ dohash: // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. @@ -225,7 +227,7 @@ func mapassign_faststr(t *maptype, h *hmap, s string) unsafe.Pointer { panic(plainError("assignment to entry in nil map")) } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_faststr)) } if h.flags&hashWriting != 0 { @@ -255,7 +257,7 @@ again: bucketloop: for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { if b.tophash[i] != top { if isEmpty(b.tophash[i]) && insertb == nil { insertb = b @@ -302,7 +304,7 @@ bucketloop: insertb = h.newoverflow(t, b) inserti = 0 // not necessary, but avoids needlessly spilling inserti } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = top // mask inserti to avoid bounds checks + insertb.tophash[inserti&(abi.OldMapBucketCount-1)] = top // mask inserti to avoid bounds checks insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*2*goarch.PtrSize) // store new key at insert position @@ -310,7 +312,7 @@ bucketloop: h.count++ done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+inserti*uintptr(t.ValueSize)) + elem := add(unsafe.Pointer(insertb), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+inserti*uintptr(t.ValueSize)) if h.flags&hashWriting == 0 { fatal("concurrent map writes") } @@ -320,7 +322,7 @@ done: func mapdelete_faststr(t *maptype, h *hmap, ky string) { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapdelete_faststr)) } if h == nil || h.count == 0 { @@ -345,7 +347,7 @@ func mapdelete_faststr(t *maptype, h *hmap, ky string) { top := tophash(hash) search: for ; b != nil; b = b.overflow(t) { - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { + for i, kptr := uintptr(0), b.keys(); i < abi.OldMapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { k := (*stringStruct)(kptr) if k.len != key.len || b.tophash[i] != top { continue @@ -355,7 +357,7 @@ search: } // Clear key's pointer. k.str = nil - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) + e := add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) if t.Elem.Pointers() { memclrHasPointers(e, t.Elem.Size_) } else { @@ -364,7 +366,7 @@ search: b.tophash[i] = emptyOne // If the bucket now ends in a bunch of emptyOne states, // change those to emptyRest states. - if i == abi.MapBucketCount-1 { + if i == abi.OldMapBucketCount-1 { if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest { goto notLast } @@ -383,7 +385,7 @@ search: c := b for b = bOrig; b.overflow(t) != c; b = b.overflow(t) { } - i = abi.MapBucketCount - 1 + i = abi.OldMapBucketCount - 1 } else { i-- } @@ -431,7 +433,7 @@ func evacuate_faststr(t *maptype, h *hmap, oldbucket uintptr) { x := &xy[0] x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.BucketSize))) x.k = add(unsafe.Pointer(x.b), dataOffset) - x.e = add(x.k, abi.MapBucketCount*2*goarch.PtrSize) + x.e = add(x.k, abi.OldMapBucketCount*2*goarch.PtrSize) if !h.sameSizeGrow() { // Only calculate y pointers if we're growing bigger. @@ -439,13 +441,13 @@ func evacuate_faststr(t *maptype, h *hmap, oldbucket uintptr) { y := &xy[1] y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.BucketSize))) y.k = add(unsafe.Pointer(y.b), dataOffset) - y.e = add(y.k, abi.MapBucketCount*2*goarch.PtrSize) + y.e = add(y.k, abi.OldMapBucketCount*2*goarch.PtrSize) } for ; b != nil; b = b.overflow(t) { k := add(unsafe.Pointer(b), dataOffset) - e := add(k, abi.MapBucketCount*2*goarch.PtrSize) - for i := 0; i < abi.MapBucketCount; i, k, e = i+1, add(k, 2*goarch.PtrSize), add(e, uintptr(t.ValueSize)) { + e := add(k, abi.OldMapBucketCount*2*goarch.PtrSize) + for i := 0; i < abi.OldMapBucketCount; i, k, e = i+1, add(k, 2*goarch.PtrSize), add(e, uintptr(t.ValueSize)) { top := b.tophash[i] if isEmpty(top) { b.tophash[i] = evacuatedEmpty @@ -467,13 +469,13 @@ func evacuate_faststr(t *maptype, h *hmap, oldbucket uintptr) { b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY, enforced in makemap dst := &xy[useY] // evacuation destination - if dst.i == abi.MapBucketCount { + if dst.i == abi.OldMapBucketCount { dst.b = h.newoverflow(t, dst.b) dst.i = 0 dst.k = add(unsafe.Pointer(dst.b), dataOffset) - dst.e = add(dst.k, abi.MapBucketCount*2*goarch.PtrSize) + dst.e = add(dst.k, abi.OldMapBucketCount*2*goarch.PtrSize) } - dst.b.tophash[dst.i&(abi.MapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check + dst.b.tophash[dst.i&(abi.OldMapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check // Copy key. *(*string)(dst.k) = *(*string)(k) diff --git a/src/runtime/map_faststr_swiss.go b/src/runtime/map_faststr_swiss.go new file mode 100644 index 00000000..23f6c1e8 --- /dev/null +++ b/src/runtime/map_faststr_swiss.go @@ -0,0 +1,44 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package runtime + +import ( + "internal/abi" + "internal/runtime/maps" + "unsafe" +) + +// Functions below pushed from internal/runtime/maps. + +//go:linkname mapaccess1_faststr +func mapaccess1_faststr(t *abi.SwissMapType, m *maps.Map, ky string) unsafe.Pointer + +// mapaccess2_faststr should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapaccess2_faststr +func mapaccess2_faststr(t *abi.SwissMapType, m *maps.Map, ky string) (unsafe.Pointer, bool) + +// mapassign_faststr should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapassign_faststr +func mapassign_faststr(t *abi.SwissMapType, m *maps.Map, s string) unsafe.Pointer + +//go:linkname mapdelete_faststr +func mapdelete_faststr(t *abi.SwissMapType, m *maps.Map, ky string) diff --git a/src/runtime/map.go b/src/runtime/map_noswiss.go similarity index 91% rename from src/runtime/map.go rename to src/runtime/map_noswiss.go index 52d56fb5..327f0c81 100644 --- a/src/runtime/map.go +++ b/src/runtime/map_noswiss.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.swissmap + package runtime // This file contains the implementation of Go's map type. @@ -57,19 +59,22 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/math" + "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) +type maptype = abi.OldMapType + const ( // Maximum number of key/elem pairs a bucket can hold. - bucketCntBits = abi.MapBucketCountBits + bucketCntBits = abi.OldMapBucketCountBits // Maximum average load of a bucket that triggers growth is bucketCnt*13/16 (about 80% full) // Because of minimum alignment rules, bucketCnt is known to be at least 8. // Represent as loadFactorNum/loadFactorDen, to allow integer math. loadFactorDen = 2 - loadFactorNum = loadFactorDen * abi.MapBucketCount * 13 / 16 + loadFactorNum = loadFactorDen * abi.OldMapBucketCount * 13 / 16 // data offset should be the size of the bmap struct, but needs to be // aligned correctly. For amd64p32 this means 64-bit alignment @@ -118,6 +123,7 @@ type hmap struct { buckets unsafe.Pointer // array of 2^B Buckets. may be nil if count==0. oldbuckets unsafe.Pointer // previous bucket array of half the size, non-nil only when growing nevacuate uintptr // progress counter for evacuation (buckets less than this have been evacuated) + clearSeq uint64 extra *mapextra // optional fields } @@ -144,7 +150,7 @@ type bmap struct { // tophash generally contains the top byte of the hash value // for each key in this bucket. If tophash[0] < minTopHash, // tophash[0] is a bucket evacuation state instead. - tophash [abi.MapBucketCount]uint8 + tophash [abi.OldMapBucketCount]uint8 // Followed by bucketCnt keys and then bucketCnt elems. // NOTE: packing all the keys together and then all the elems together makes the // code a bit more complicated than alternating key/elem/key/elem/... but it allows @@ -171,6 +177,7 @@ type hiter struct { i uint8 bucket uintptr checkBucket uintptr + clearSeq uint64 } // bucketShift returns 1<> h.B & (abi.MapBucketCount - 1)) + it.offset = uint8(r >> h.B & (abi.OldMapBucketCount - 1)) // iterator state it.bucket = it.startBucket @@ -924,7 +929,6 @@ func mapiterinit(t *maptype, h *hmap, it *hiter) { // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/RomiChan/protobuf // - github.com/segmentio/encoding // - github.com/ugorji/go/codec @@ -937,7 +941,7 @@ func mapiterinit(t *maptype, h *hmap, it *hiter) { func mapiternext(it *hiter) { h := it.h if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapiternext)) } if h.flags&hashWriting != 0 { @@ -981,8 +985,8 @@ next: } i = 0 } - for ; i < abi.MapBucketCount; i++ { - offi := (i + it.offset) & (abi.MapBucketCount - 1) + for ; i < abi.OldMapBucketCount; i++ { + offi := (i + it.offset) & (abi.OldMapBucketCount - 1) if isEmpty(b.tophash[offi]) || b.tophash[offi] == evacuatedEmpty { // TODO: emptyRest is hard to use here, as we start iterating // in the middle of a bucket. It's feasible, just tricky. @@ -992,7 +996,7 @@ next: if t.IndirectKey() { k = *((*unsafe.Pointer)(k)) } - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+uintptr(offi)*uintptr(t.ValueSize)) + e := add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*uintptr(t.KeySize)+uintptr(offi)*uintptr(t.ValueSize)) if checkBucket != noCheck && !h.sameSizeGrow() { // Special case: iterator was started during a grow to a larger size // and the grow is not done yet. We're working on a bucket whose @@ -1021,8 +1025,9 @@ next: } } } - if (b.tophash[offi] != evacuatedX && b.tophash[offi] != evacuatedY) || - !(t.ReflexiveKey() || t.Key.Equal(k, k)) { + if it.clearSeq == h.clearSeq && + ((b.tophash[offi] != evacuatedX && b.tophash[offi] != evacuatedY) || + !(t.ReflexiveKey() || t.Key.Equal(k, k))) { // This is the golden data, we can return it. // OR // key!=key, so the entry can't be deleted or updated, so we can just return it. @@ -1062,19 +1067,9 @@ next: // mapclear deletes all keys from a map. // It is called by the compiler. -// -// mapclear should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname mapclear func mapclear(t *maptype, h *hmap) { if raceenabled && h != nil { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(mapclear) racewritepc(unsafe.Pointer(h), callerpc, pc) } @@ -1088,28 +1083,12 @@ func mapclear(t *maptype, h *hmap) { } h.flags ^= hashWriting - - // Mark buckets empty, so existing iterators can be terminated, see issue #59411. - markBucketsEmpty := func(bucket unsafe.Pointer, mask uintptr) { - for i := uintptr(0); i <= mask; i++ { - b := (*bmap)(add(bucket, i*uintptr(t.BucketSize))) - for ; b != nil; b = b.overflow(t) { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - b.tophash[i] = emptyRest - } - } - } - } - markBucketsEmpty(h.buckets, bucketMask(h.B)) - if oldBuckets := h.oldbuckets; oldBuckets != nil { - markBucketsEmpty(oldBuckets, h.oldbucketmask()) - } - h.flags &^= sameSizeGrow h.oldbuckets = nil h.nevacuate = 0 h.noverflow = 0 h.count = 0 + h.clearSeq++ // Reset the hash seed to make it more difficult for attackers to // repeatedly trigger hash collisions. See issue 25237. @@ -1181,7 +1160,7 @@ func hashGrow(t *maptype, h *hmap) { // overLoadFactor reports whether count items placed in 1< abi.MapBucketCount && uintptr(count) > loadFactorNum*(bucketShift(B)/loadFactorDen) + return count > abi.OldMapBucketCount && uintptr(count) > loadFactorNum*(bucketShift(B)/loadFactorDen) } // tooManyOverflowBuckets reports whether noverflow buckets is too many for a map with 1< abi.MapMaxKeyBytes && (!t.IndirectKey() || t.KeySize != uint8(goarch.PtrSize)) || - t.Key.Size_ <= abi.MapMaxKeyBytes && (t.IndirectKey() || t.KeySize != uint8(t.Key.Size_)) { + if t.Key.Size_ > abi.OldMapMaxKeyBytes && (!t.IndirectKey() || t.KeySize != uint8(goarch.PtrSize)) || + t.Key.Size_ <= abi.OldMapMaxKeyBytes && (t.IndirectKey() || t.KeySize != uint8(t.Key.Size_)) { throw("key size wrong") } - if t.Elem.Size_ > abi.MapMaxElemBytes && (!t.IndirectElem() || t.ValueSize != uint8(goarch.PtrSize)) || - t.Elem.Size_ <= abi.MapMaxElemBytes && (t.IndirectElem() || t.ValueSize != uint8(t.Elem.Size_)) { + if t.Elem.Size_ > abi.OldMapMaxElemBytes && (!t.IndirectElem() || t.ValueSize != uint8(goarch.PtrSize)) || + t.Elem.Size_ <= abi.OldMapMaxElemBytes && (t.IndirectElem() || t.ValueSize != uint8(t.Elem.Size_)) { throw("elem size wrong") } - if t.Key.Align_ > abi.MapBucketCount { + if t.Key.Align_ > abi.OldMapBucketCount { throw("key align too big") } - if t.Elem.Align_ > abi.MapBucketCount { + if t.Elem.Align_ > abi.OldMapBucketCount { throw("elem align too big") } if t.Key.Size_%uintptr(t.Key.Align_) != 0 { @@ -1431,7 +1410,7 @@ func reflect_makemap(t *maptype, cap int) *hmap { if t.Elem.Size_%uintptr(t.Elem.Align_) != 0 { throw("elem size not a multiple of elem align") } - if abi.MapBucketCount < 8 { + if abi.OldMapBucketCount < 8 { throw("bucketsize too small for proper alignment") } if dataOffset%uintptr(t.Key.Align_) != 0 { @@ -1537,7 +1516,7 @@ func reflect_mapiternext(it *hiter) { mapiternext(it) } -// reflect_mapiterkey is for package reflect, +// reflect_mapiterkey was for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/goccy/go-json @@ -1551,7 +1530,7 @@ func reflect_mapiterkey(it *hiter) unsafe.Pointer { return it.key } -// reflect_mapiterelem is for package reflect, +// reflect_mapiterelem was for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/goccy/go-json @@ -1580,7 +1559,7 @@ func reflect_maplen(h *hmap) int { return 0 } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(reflect_maplen)) } return h.count @@ -1597,7 +1576,7 @@ func reflectlite_maplen(h *hmap) int { return 0 } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(reflect_maplen)) } return h.count @@ -1622,26 +1601,26 @@ func mapclone(m any) any { // moveToBmap moves a bucket from src to dst. It returns the destination bucket or new destination bucket if it overflows // and the pos that the next key/value will be written, if pos == bucketCnt means needs to written in overflow bucket. func moveToBmap(t *maptype, h *hmap, dst *bmap, pos int, src *bmap) (*bmap, int) { - for i := 0; i < abi.MapBucketCount; i++ { + for i := 0; i < abi.OldMapBucketCount; i++ { if isEmpty(src.tophash[i]) { continue } - for ; pos < abi.MapBucketCount; pos++ { + for ; pos < abi.OldMapBucketCount; pos++ { if isEmpty(dst.tophash[pos]) { break } } - if pos == abi.MapBucketCount { + if pos == abi.OldMapBucketCount { dst = h.newoverflow(t, dst) pos = 0 } srcK := add(unsafe.Pointer(src), dataOffset+uintptr(i)*uintptr(t.KeySize)) - srcEle := add(unsafe.Pointer(src), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+uintptr(i)*uintptr(t.ValueSize)) + srcEle := add(unsafe.Pointer(src), dataOffset+abi.OldMapBucketCount*uintptr(t.KeySize)+uintptr(i)*uintptr(t.ValueSize)) dstK := add(unsafe.Pointer(dst), dataOffset+uintptr(pos)*uintptr(t.KeySize)) - dstEle := add(unsafe.Pointer(dst), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+uintptr(pos)*uintptr(t.ValueSize)) + dstEle := add(unsafe.Pointer(dst), dataOffset+abi.OldMapBucketCount*uintptr(t.KeySize)+uintptr(pos)*uintptr(t.ValueSize)) dst.tophash[pos] = src.tophash[i] if t.IndirectKey() { @@ -1754,7 +1733,7 @@ func mapclone2(t *maptype, src *hmap) *hmap { // Process entries one at a time. for srcBmap != nil { // move from oldBlucket to new bucket - for i := uintptr(0); i < abi.MapBucketCount; i++ { + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { if isEmpty(srcBmap.tophash[i]) { continue } @@ -1768,7 +1747,7 @@ func mapclone2(t *maptype, src *hmap) *hmap { srcK = *((*unsafe.Pointer)(srcK)) } - srcEle := add(unsafe.Pointer(srcBmap), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) + srcEle := add(unsafe.Pointer(srcBmap), dataOffset+abi.OldMapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) if t.IndirectElem() { srcEle = *((*unsafe.Pointer)(srcEle)) } @@ -1794,7 +1773,7 @@ func keys(m any, p unsafe.Pointer) { } s := (*slice)(p) r := int(rand()) - offset := uint8(r >> h.B & (abi.MapBucketCount - 1)) + offset := uint8(r >> h.B & (abi.OldMapBucketCount - 1)) if h.B == 0 { copyKeys(t, h, (*bmap)(h.buckets), s, offset) return @@ -1823,8 +1802,8 @@ func keys(m any, p unsafe.Pointer) { func copyKeys(t *maptype, h *hmap, b *bmap, s *slice, offset uint8) { for b != nil { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - offi := (i + uintptr(offset)) & (abi.MapBucketCount - 1) + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { + offi := (i + uintptr(offset)) & (abi.OldMapBucketCount - 1) if isEmpty(b.tophash[offi]) { continue } @@ -1857,7 +1836,7 @@ func values(m any, p unsafe.Pointer) { } s := (*slice)(p) r := int(rand()) - offset := uint8(r >> h.B & (abi.MapBucketCount - 1)) + offset := uint8(r >> h.B & (abi.OldMapBucketCount - 1)) if h.B == 0 { copyValues(t, h, (*bmap)(h.buckets), s, offset) return @@ -1886,8 +1865,8 @@ func values(m any, p unsafe.Pointer) { func copyValues(t *maptype, h *hmap, b *bmap, s *slice, offset uint8) { for b != nil { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - offi := (i + uintptr(offset)) & (abi.MapBucketCount - 1) + for i := uintptr(0); i < abi.OldMapBucketCount; i++ { + offi := (i + uintptr(offset)) & (abi.OldMapBucketCount - 1) if isEmpty(b.tophash[offi]) { continue } @@ -1896,7 +1875,7 @@ func copyValues(t *maptype, h *hmap, b *bmap, s *slice, offset uint8) { fatal("concurrent map read and map write") } - ele := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+offi*uintptr(t.ValueSize)) + ele := add(unsafe.Pointer(b), dataOffset+abi.OldMapBucketCount*uintptr(t.KeySize)+offi*uintptr(t.ValueSize)) if t.IndirectElem() { ele = *((*unsafe.Pointer)(ele)) } diff --git a/src/runtime/map_noswiss_test.go b/src/runtime/map_noswiss_test.go new file mode 100644 index 00000000..5af7b7b8 --- /dev/null +++ b/src/runtime/map_noswiss_test.go @@ -0,0 +1,214 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.swissmap + +package runtime_test + +import ( + "internal/abi" + "internal/goarch" + "runtime" + "slices" + "testing" +) + +func TestHmapSize(t *testing.T) { + // The structure of hmap is defined in runtime/map.go + // and in cmd/compile/internal/reflectdata/map.go and must be in sync. + // The size of hmap should be 56 bytes on 64 bit and 36 bytes on 32 bit platforms. + var hmapSize = uintptr(2*8 + 5*goarch.PtrSize) + if runtime.RuntimeHmapSize != hmapSize { + t.Errorf("sizeof(runtime.hmap{})==%d, want %d", runtime.RuntimeHmapSize, hmapSize) + } +} + +func TestLoadFactor(t *testing.T) { + for b := uint8(0); b < 20; b++ { + count := 13 * (1 << b) / 2 // 6.5 + if b == 0 { + count = 8 + } + if runtime.OverLoadFactor(count, b) { + t.Errorf("OverLoadFactor(%d,%d)=true, want false", count, b) + } + if !runtime.OverLoadFactor(count+1, b) { + t.Errorf("OverLoadFactor(%d,%d)=false, want true", count+1, b) + } + } +} + +func TestMapIterOrder(t *testing.T) { + sizes := []int{3, 7, 9, 15} + if abi.OldMapBucketCountBits >= 5 { + // it gets flaky (often only one iteration order) at size 3 when abi.MapBucketCountBits >=5. + t.Fatalf("This test becomes flaky if abi.MapBucketCountBits(=%d) is 5 or larger", abi.OldMapBucketCountBits) + } + for _, n := range sizes { + for i := 0; i < 1000; i++ { + // Make m be {0: true, 1: true, ..., n-1: true}. + m := make(map[int]bool) + for i := 0; i < n; i++ { + m[i] = true + } + // Check that iterating over the map produces at least two different orderings. + ord := func() []int { + var s []int + for key := range m { + s = append(s, key) + } + return s + } + first := ord() + ok := false + for try := 0; try < 100; try++ { + if !slices.Equal(first, ord()) { + ok = true + break + } + } + if !ok { + t.Errorf("Map with n=%d elements had consistent iteration order: %v", n, first) + break + } + } + } +} + +const bs = abi.OldMapBucketCount + +// belowOverflow should be a pretty-full pair of buckets; +// atOverflow is 1/8 bs larger = 13/8 buckets or two buckets +// that are 13/16 full each, which is the overflow boundary. +// Adding one to that should ensure overflow to the next higher size. +const ( + belowOverflow = bs * 3 / 2 // 1.5 bs = 2 buckets @ 75% + atOverflow = belowOverflow + bs/8 // 2 buckets at 13/16 fill. +) + +var mapBucketTests = [...]struct { + n int // n is the number of map elements + noescape int // number of expected buckets for non-escaping map + escape int // number of expected buckets for escaping map +}{ + {-(1 << 30), 1, 1}, + {-1, 1, 1}, + {0, 1, 1}, + {1, 1, 1}, + {bs, 1, 1}, + {bs + 1, 2, 2}, + {belowOverflow, 2, 2}, // 1.5 bs = 2 buckets @ 75% + {atOverflow + 1, 4, 4}, // 13/8 bs + 1 == overflow to 4 + + {2 * belowOverflow, 4, 4}, // 3 bs = 4 buckets @75% + {2*atOverflow + 1, 8, 8}, // 13/4 bs + 1 = overflow to 8 + + {4 * belowOverflow, 8, 8}, // 6 bs = 8 buckets @ 75% + {4*atOverflow + 1, 16, 16}, // 13/2 bs + 1 = overflow to 16 +} + +func TestMapBuckets(t *testing.T) { + // Test that maps of different sizes have the right number of buckets. + // Non-escaping maps with small buckets (like map[int]int) never + // have a nil bucket pointer due to starting with preallocated buckets + // on the stack. Escaping maps start with a non-nil bucket pointer if + // hint size is above bucketCnt and thereby have more than one bucket. + // These tests depend on bucketCnt and loadFactor* in map.go. + t.Run("mapliteral", func(t *testing.T) { + for _, tt := range mapBucketTests { + localMap := map[int]int{} + if runtime.MapBucketsPointerIsNil(localMap) { + t.Errorf("no escape: buckets pointer is nil for non-escaping map") + } + for i := 0; i < tt.n; i++ { + localMap[i] = i + } + if got := runtime.MapBucketsCount(localMap); got != tt.noescape { + t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) + } + escapingMap := runtime.Escape(map[int]int{}) + if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { + t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) + } + for i := 0; i < tt.n; i++ { + escapingMap[i] = i + } + if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { + t.Errorf("escape n=%d want %d buckets, got %d", tt.n, tt.escape, got) + } + } + }) + t.Run("nohint", func(t *testing.T) { + for _, tt := range mapBucketTests { + localMap := make(map[int]int) + if runtime.MapBucketsPointerIsNil(localMap) { + t.Errorf("no escape: buckets pointer is nil for non-escaping map") + } + for i := 0; i < tt.n; i++ { + localMap[i] = i + } + if got := runtime.MapBucketsCount(localMap); got != tt.noescape { + t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) + } + escapingMap := runtime.Escape(make(map[int]int)) + if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { + t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) + } + for i := 0; i < tt.n; i++ { + escapingMap[i] = i + } + if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { + t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) + } + } + }) + t.Run("makemap", func(t *testing.T) { + for _, tt := range mapBucketTests { + localMap := make(map[int]int, tt.n) + if runtime.MapBucketsPointerIsNil(localMap) { + t.Errorf("no escape: buckets pointer is nil for non-escaping map") + } + for i := 0; i < tt.n; i++ { + localMap[i] = i + } + if got := runtime.MapBucketsCount(localMap); got != tt.noescape { + t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) + } + escapingMap := runtime.Escape(make(map[int]int, tt.n)) + if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { + t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) + } + for i := 0; i < tt.n; i++ { + escapingMap[i] = i + } + if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { + t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) + } + } + }) + t.Run("makemap64", func(t *testing.T) { + for _, tt := range mapBucketTests { + localMap := make(map[int]int, int64(tt.n)) + if runtime.MapBucketsPointerIsNil(localMap) { + t.Errorf("no escape: buckets pointer is nil for non-escaping map") + } + for i := 0; i < tt.n; i++ { + localMap[i] = i + } + if got := runtime.MapBucketsCount(localMap); got != tt.noescape { + t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) + } + escapingMap := runtime.Escape(make(map[int]int, tt.n)) + if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { + t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) + } + for i := 0; i < tt.n; i++ { + escapingMap[i] = i + } + if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { + t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) + } + } + }) +} diff --git a/src/runtime/map_swiss.go b/src/runtime/map_swiss.go new file mode 100644 index 00000000..a8fe8725 --- /dev/null +++ b/src/runtime/map_swiss.go @@ -0,0 +1,363 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package runtime + +import ( + "internal/abi" + "internal/runtime/maps" + "internal/runtime/sys" + "unsafe" +) + +const ( + // TODO: remove? These are used by tests but not the actual map + loadFactorNum = 7 + loadFactorDen = 8 +) + +type maptype = abi.SwissMapType + +//go:linkname maps_errNilAssign internal/runtime/maps.errNilAssign +var maps_errNilAssign error = plainError("assignment to entry in nil map") + +//go:linkname maps_mapKeyError internal/runtime/maps.mapKeyError +func maps_mapKeyError(t *abi.SwissMapType, p unsafe.Pointer) error { + return mapKeyError(t, p) +} + +func makemap64(t *abi.SwissMapType, hint int64, m *maps.Map) *maps.Map { + if int64(int(hint)) != hint { + hint = 0 + } + return makemap(t, int(hint), m) +} + +// makemap_small implements Go map creation for make(map[k]v) and +// make(map[k]v, hint) when hint is known to be at most abi.SwissMapGroupSlots +// at compile time and the map needs to be allocated on the heap. +// +// makemap_small should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname makemap_small +func makemap_small() *maps.Map { + return maps.NewEmptyMap() +} + +// makemap implements Go map creation for make(map[k]v, hint). +// If the compiler has determined that the map or the first group +// can be created on the stack, m and optionally m.dirPtr may be non-nil. +// If m != nil, the map can be created directly in m. +// If m.dirPtr != nil, it points to a group usable for a small map. +// +// makemap should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname makemap +func makemap(t *abi.SwissMapType, hint int, m *maps.Map) *maps.Map { + if hint < 0 { + hint = 0 + } + + return maps.NewMap(t, uintptr(hint), m, maxAlloc) +} + +// mapaccess1 returns a pointer to h[key]. Never returns nil, instead +// it will return a reference to the zero object for the elem type if +// the key is not in the map. +// NOTE: The returned pointer may keep the whole map live, so don't +// hold onto it for very long. +// +// mapaccess1 is pushed from internal/runtime/maps. We could just call it, but +// we want to avoid one layer of call. +// +//go:linkname mapaccess1 +func mapaccess1(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer + +// mapaccess2 should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapaccess2 +func mapaccess2(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) (unsafe.Pointer, bool) + +func mapaccess1_fat(t *abi.SwissMapType, m *maps.Map, key, zero unsafe.Pointer) unsafe.Pointer { + e := mapaccess1(t, m, key) + if e == unsafe.Pointer(&zeroVal[0]) { + return zero + } + return e +} + +func mapaccess2_fat(t *abi.SwissMapType, m *maps.Map, key, zero unsafe.Pointer) (unsafe.Pointer, bool) { + e := mapaccess1(t, m, key) + if e == unsafe.Pointer(&zeroVal[0]) { + return zero, false + } + return e, true +} + +// mapassign is pushed from internal/runtime/maps. We could just call it, but +// we want to avoid one layer of call. +// +// mapassign should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/RomiChan/protobuf +// - github.com/segmentio/encoding +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapassign +func mapassign(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer + +// mapdelete should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/ugorji/go/codec +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapdelete +func mapdelete(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(mapdelete) + racewritepc(unsafe.Pointer(m), callerpc, pc) + raceReadObjectPC(t.Key, key, callerpc, pc) + } + if msanenabled && m != nil { + msanread(key, t.Key.Size_) + } + if asanenabled && m != nil { + asanread(key, t.Key.Size_) + } + + m.Delete(t, key) +} + +// mapIterStart initializes the Iter struct used for ranging over maps and +// performs the first step of iteration. The Iter struct pointed to by 'it' is +// allocated on the stack by the compilers order pass or on the heap by +// reflect. Both need to have zeroed it since the struct contains pointers. +func mapIterStart(t *abi.SwissMapType, m *maps.Map, it *maps.Iter) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(mapIterStart)) + } + + it.Init(t, m) + it.Next() +} + +// mapIterNext performs the next step of iteration. Afterwards, the next +// key/elem are in it.Key()/it.Elem(). +func mapIterNext(it *maps.Iter) { + if raceenabled { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(it.Map()), callerpc, abi.FuncPCABIInternal(mapIterNext)) + } + + it.Next() +} + +// mapclear deletes all keys from a map. +func mapclear(t *abi.SwissMapType, m *maps.Map) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(mapclear) + racewritepc(unsafe.Pointer(m), callerpc, pc) + } + + m.Clear(t) +} + +// Reflect stubs. Called from ../reflect/asm_*.s + +// reflect_makemap is for package reflect, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/modern-go/reflect2 +// - github.com/goccy/go-json +// - github.com/RomiChan/protobuf +// - github.com/segmentio/encoding +// - github.com/v2pro/plz +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_makemap reflect.makemap +func reflect_makemap(t *abi.SwissMapType, cap int) *maps.Map { + // Check invariants and reflects math. + if t.Key.Equal == nil { + throw("runtime.reflect_makemap: unsupported map key type") + } + // TODO: other checks + + return makemap(t, cap, nil) +} + +// reflect_mapaccess is for package reflect, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/modern-go/reflect2 +// - github.com/v2pro/plz +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapaccess reflect.mapaccess +func reflect_mapaccess(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer { + elem, ok := mapaccess2(t, m, key) + if !ok { + // reflect wants nil for a missing element + elem = nil + } + return elem +} + +//go:linkname reflect_mapaccess_faststr reflect.mapaccess_faststr +func reflect_mapaccess_faststr(t *abi.SwissMapType, m *maps.Map, key string) unsafe.Pointer { + elem, ok := mapaccess2_faststr(t, m, key) + if !ok { + // reflect wants nil for a missing element + elem = nil + } + return elem +} + +// reflect_mapassign is for package reflect, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/v2pro/plz +// +// Do not remove or change the type signature. +// +//go:linkname reflect_mapassign reflect.mapassign0 +func reflect_mapassign(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer, elem unsafe.Pointer) { + p := mapassign(t, m, key) + typedmemmove(t.Elem, p, elem) +} + +//go:linkname reflect_mapassign_faststr reflect.mapassign_faststr0 +func reflect_mapassign_faststr(t *abi.SwissMapType, m *maps.Map, key string, elem unsafe.Pointer) { + p := mapassign_faststr(t, m, key) + typedmemmove(t.Elem, p, elem) +} + +//go:linkname reflect_mapdelete reflect.mapdelete +func reflect_mapdelete(t *abi.SwissMapType, m *maps.Map, key unsafe.Pointer) { + mapdelete(t, m, key) +} + +//go:linkname reflect_mapdelete_faststr reflect.mapdelete_faststr +func reflect_mapdelete_faststr(t *abi.SwissMapType, m *maps.Map, key string) { + mapdelete_faststr(t, m, key) +} + +// reflect_maplen is for package reflect, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/goccy/go-json +// - github.com/wI2L/jettison +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_maplen reflect.maplen +func reflect_maplen(m *maps.Map) int { + if m == nil { + return 0 + } + if raceenabled { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(reflect_maplen)) + } + return int(m.Used()) +} + +//go:linkname reflect_mapclear reflect.mapclear +func reflect_mapclear(t *abi.SwissMapType, m *maps.Map) { + mapclear(t, m) +} + +//go:linkname reflectlite_maplen internal/reflectlite.maplen +func reflectlite_maplen(m *maps.Map) int { + if m == nil { + return 0 + } + if raceenabled { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(reflect_maplen)) + } + return int(m.Used()) +} + +// mapinitnoop is a no-op function known the Go linker; if a given global +// map (of the right size) is determined to be dead, the linker will +// rewrite the relocation (from the package init func) from the outlined +// map init function to this symbol. Defined in assembly so as to avoid +// complications with instrumentation (coverage, etc). +func mapinitnoop() + +// mapclone for implementing maps.Clone +// +//go:linkname mapclone maps.clone +func mapclone(m any) any { + e := efaceOf(&m) + e.data = unsafe.Pointer(mapclone2((*abi.SwissMapType)(unsafe.Pointer(e._type)), (*maps.Map)(e.data))) + return m +} + +func mapclone2(t *abi.SwissMapType, src *maps.Map) *maps.Map { + dst := makemap(t, int(src.Used()), nil) + + var iter maps.Iter + iter.Init(t, src) + for iter.Next(); iter.Key() != nil; iter.Next() { + dst.Put(t, iter.Key(), iter.Elem()) + } + + return dst +} + +// keys for implementing maps.keys +// +//go:linkname keys maps.keys +func keys(m any, p unsafe.Pointer) { + // Currently unused in the maps package. + panic("unimplemented") +} + +// values for implementing maps.values +// +//go:linkname values maps.values +func values(m any, p unsafe.Pointer) { + // Currently unused in the maps package. + panic("unimplemented") +} diff --git a/src/runtime/map_swiss_test.go b/src/runtime/map_swiss_test.go new file mode 100644 index 00000000..d5c9fdbe --- /dev/null +++ b/src/runtime/map_swiss_test.go @@ -0,0 +1,75 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.swissmap + +package runtime_test + +import ( + "internal/abi" + "internal/goarch" + "internal/runtime/maps" + "slices" + "testing" + "unsafe" +) + +func TestHmapSize(t *testing.T) { + // The structure of Map is defined in internal/runtime/maps/map.go + // and in cmd/compile/internal/reflectdata/map_swiss.go and must be in sync. + // The size of Map should be 48 bytes on 64 bit and 32 bytes on 32 bit platforms. + wantSize := uintptr(2*8 + 4*goarch.PtrSize) + gotSize := unsafe.Sizeof(maps.Map{}) + if gotSize != wantSize { + t.Errorf("sizeof(maps.Map{})==%d, want %d", gotSize, wantSize) + } +} + +// See also reflect_test.TestGroupSizeZero. +func TestGroupSizeZero(t *testing.T) { + var m map[struct{}]struct{} + mTyp := abi.TypeOf(m) + mt := (*abi.SwissMapType)(unsafe.Pointer(mTyp)) + + // internal/runtime/maps when create pointers to slots, even if slots + // are size 0. The compiler should have reserved an extra word to + // ensure that pointers to the zero-size type at the end of group are + // valid. + if mt.Group.Size() <= 8 { + t.Errorf("Group size got %d want >8", mt.Group.Size()) + } +} + +func TestMapIterOrder(t *testing.T) { + sizes := []int{3, 7, 9, 15} + for _, n := range sizes { + for i := 0; i < 1000; i++ { + // Make m be {0: true, 1: true, ..., n-1: true}. + m := make(map[int]bool) + for i := 0; i < n; i++ { + m[i] = true + } + // Check that iterating over the map produces at least two different orderings. + ord := func() []int { + var s []int + for key := range m { + s = append(s, key) + } + return s + } + first := ord() + ok := false + for try := 0; try < 100; try++ { + if !slices.Equal(first, ord()) { + ok = true + break + } + } + if !ok { + t.Errorf("Map with n=%d elements had consistent iteration order: %v", n, first) + break + } + } + } +} diff --git a/src/runtime/map_test.go b/src/runtime/map_test.go index 13624e09..c522c44a 100644 --- a/src/runtime/map_test.go +++ b/src/runtime/map_test.go @@ -6,8 +6,7 @@ package runtime_test import ( "fmt" - "internal/abi" - "internal/goarch" + "internal/goexperiment" "internal/testenv" "math" "os" @@ -21,17 +20,6 @@ import ( "unsafe" ) -func TestHmapSize(t *testing.T) { - // The structure of hmap is defined in runtime/map.go - // and in cmd/compile/internal/gc/reflect.go and must be in sync. - // The size of hmap should be 48 bytes on 64 bit and 28 bytes on 32 bit platforms. - var hmapSize = uintptr(8 + 5*goarch.PtrSize) - if runtime.RuntimeHmapSize != hmapSize { - t.Errorf("sizeof(runtime.hmap{})==%d, want %d", runtime.RuntimeHmapSize, hmapSize) - } - -} - // negative zero is a good test because: // 1. 0 and -0 are equal, yet have distinct representations. // 2. 0 is represented as all zeros, -0 isn't. @@ -144,7 +132,7 @@ func TestMapAppendAssignment(t *testing.T) { m[0] = append(m[0], a...) want := []int{12345, 67890, 123, 456, 7, 8, 9, 0} - if got := m[0]; !reflect.DeepEqual(got, want) { + if got := m[0]; !slices.Equal(got, want) { t.Errorf("got %v, want %v", got, want) } } @@ -431,6 +419,12 @@ func TestEmptyKeyAndValue(t *testing.T) { if len(a) != 1 { t.Errorf("empty value insert problem") } + if len(b) != 1 { + t.Errorf("empty key insert problem") + } + if len(c) != 1 { + t.Errorf("empty key+value insert problem") + } if b[empty{}] != 1 { t.Errorf("empty key returned wrong value") } @@ -509,43 +503,6 @@ func TestMapNanGrowIterator(t *testing.T) { } } -func TestMapIterOrder(t *testing.T) { - sizes := []int{3, 7, 9, 15} - if abi.MapBucketCountBits >= 5 { - // it gets flaky (often only one iteration order) at size 3 when abi.MapBucketCountBits >=5. - t.Fatalf("This test becomes flaky if abi.MapBucketCountBits(=%d) is 5 or larger", abi.MapBucketCountBits) - } - for _, n := range sizes { - for i := 0; i < 1000; i++ { - // Make m be {0: true, 1: true, ..., n-1: true}. - m := make(map[int]bool) - for i := 0; i < n; i++ { - m[i] = true - } - // Check that iterating over the map produces at least two different orderings. - ord := func() []int { - var s []int - for key := range m { - s = append(s, key) - } - return s - } - first := ord() - ok := false - for try := 0; try < 100; try++ { - if !reflect.DeepEqual(first, ord()) { - ok = true - break - } - } - if !ok { - t.Errorf("Map with n=%d elements had consistent iteration order: %v", n, first) - break - } - } - } -} - // Issue 8410 func TestMapSparseIterOrder(t *testing.T) { // Run several rounds to increase the probability @@ -582,6 +539,38 @@ NextRound: } } +// Map iteration must not return duplicate entries. +func TestMapIterDuplicate(t *testing.T) { + // Run several rounds to increase the probability + // of failure. One is not enough. + for range 1000 { + m := make(map[int]bool) + // Add 1000 items, remove 980. + for i := 0; i < 1000; i++ { + m[i] = true + } + for i := 20; i < 1000; i++ { + delete(m, i) + } + + var want []int + for i := 0; i < 20; i++ { + want = append(want, i) + } + + var got []int + for i := range m { + got = append(got, i) + } + + slices.Sort(got) + + if !reflect.DeepEqual(got, want) { + t.Errorf("iteration got %v want %v\n", got, want) + } + } +} + func TestMapStringBytesLookup(t *testing.T) { // Use large string keys to avoid small-allocation coalescing, // which can cause AllocsPerRun to report lower counts than it should. @@ -682,165 +671,6 @@ func TestIgnoreBogusMapHint(t *testing.T) { } } -const bs = abi.MapBucketCount - -// belowOverflow should be a pretty-full pair of buckets; -// atOverflow is 1/8 bs larger = 13/8 buckets or two buckets -// that are 13/16 full each, which is the overflow boundary. -// Adding one to that should ensure overflow to the next higher size. -const ( - belowOverflow = bs * 3 / 2 // 1.5 bs = 2 buckets @ 75% - atOverflow = belowOverflow + bs/8 // 2 buckets at 13/16 fill. -) - -var mapBucketTests = [...]struct { - n int // n is the number of map elements - noescape int // number of expected buckets for non-escaping map - escape int // number of expected buckets for escaping map -}{ - {-(1 << 30), 1, 1}, - {-1, 1, 1}, - {0, 1, 1}, - {1, 1, 1}, - {bs, 1, 1}, - {bs + 1, 2, 2}, - {belowOverflow, 2, 2}, // 1.5 bs = 2 buckets @ 75% - {atOverflow + 1, 4, 4}, // 13/8 bs + 1 == overflow to 4 - - {2 * belowOverflow, 4, 4}, // 3 bs = 4 buckets @75% - {2*atOverflow + 1, 8, 8}, // 13/4 bs + 1 = overflow to 8 - - {4 * belowOverflow, 8, 8}, // 6 bs = 8 buckets @ 75% - {4*atOverflow + 1, 16, 16}, // 13/2 bs + 1 = overflow to 16 -} - -func TestMapBuckets(t *testing.T) { - // Test that maps of different sizes have the right number of buckets. - // Non-escaping maps with small buckets (like map[int]int) never - // have a nil bucket pointer due to starting with preallocated buckets - // on the stack. Escaping maps start with a non-nil bucket pointer if - // hint size is above bucketCnt and thereby have more than one bucket. - // These tests depend on bucketCnt and loadFactor* in map.go. - t.Run("mapliteral", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := map[int]int{} - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(map[int]int{}) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - t.Run("nohint", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := make(map[int]int) - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(make(map[int]int)) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - t.Run("makemap", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := make(map[int]int, tt.n) - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(make(map[int]int, tt.n)) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - t.Run("makemap64", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := make(map[int]int, int64(tt.n)) - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(make(map[int]int, tt.n)) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - -} - -func benchmarkMapPop(b *testing.B, n int) { - m := map[int]int{} - for i := 0; i < b.N; i++ { - for j := 0; j < n; j++ { - m[j] = j - } - for j := 0; j < n; j++ { - // Use iterator to pop an element. - // We want this to be fast, see issue 8412. - for k := range m { - delete(m, k) - break - } - } - } -} - -func BenchmarkMapPop100(b *testing.B) { benchmarkMapPop(b, 100) } -func BenchmarkMapPop1000(b *testing.B) { benchmarkMapPop(b, 1000) } -func BenchmarkMapPop10000(b *testing.B) { benchmarkMapPop(b, 10000) } - var testNonEscapingMapVariable int = 8 func TestNonEscapingMap(t *testing.T) { @@ -849,224 +679,32 @@ func TestNonEscapingMap(t *testing.T) { m[0] = 0 }) if n != 0 { - t.Fatalf("mapliteral: want 0 allocs, got %v", n) + t.Errorf("mapliteral: want 0 allocs, got %v", n) } n = testing.AllocsPerRun(1000, func() { m := make(map[int]int) m[0] = 0 }) if n != 0 { - t.Fatalf("no hint: want 0 allocs, got %v", n) + t.Errorf("no hint: want 0 allocs, got %v", n) } n = testing.AllocsPerRun(1000, func() { m := make(map[int]int, 8) m[0] = 0 }) if n != 0 { - t.Fatalf("with small hint: want 0 allocs, got %v", n) + t.Errorf("with small hint: want 0 allocs, got %v", n) } n = testing.AllocsPerRun(1000, func() { m := make(map[int]int, testNonEscapingMapVariable) m[0] = 0 }) if n != 0 { - t.Fatalf("with variable hint: want 0 allocs, got %v", n) + t.Errorf("with variable hint: want 0 allocs, got %v", n) } } -func benchmarkMapAssignInt32(b *testing.B, n int) { - a := make(map[int32]int) - for i := 0; i < b.N; i++ { - a[int32(i&(n-1))] = i - } -} - -func benchmarkMapOperatorAssignInt32(b *testing.B, n int) { - a := make(map[int32]int) - for i := 0; i < b.N; i++ { - a[int32(i&(n-1))] += i - } -} - -func benchmarkMapAppendAssignInt32(b *testing.B, n int) { - a := make(map[int32][]int) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := int32(i & (n - 1)) - a[key] = append(a[key], i) - } -} - -func benchmarkMapDeleteInt32(b *testing.B, n int) { - a := make(map[int32]int, n) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := i; j < i+n; j++ { - a[int32(j)] = j - } - b.StartTimer() - } - delete(a, int32(i)) - } -} - -func benchmarkMapAssignInt64(b *testing.B, n int) { - a := make(map[int64]int) - for i := 0; i < b.N; i++ { - a[int64(i&(n-1))] = i - } -} - -func benchmarkMapOperatorAssignInt64(b *testing.B, n int) { - a := make(map[int64]int) - for i := 0; i < b.N; i++ { - a[int64(i&(n-1))] += i - } -} - -func benchmarkMapAppendAssignInt64(b *testing.B, n int) { - a := make(map[int64][]int) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := int64(i & (n - 1)) - a[key] = append(a[key], i) - } -} - -func benchmarkMapDeleteInt64(b *testing.B, n int) { - a := make(map[int64]int, n) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := i; j < i+n; j++ { - a[int64(j)] = j - } - b.StartTimer() - } - delete(a, int64(i)) - } -} - -func benchmarkMapAssignStr(b *testing.B, n int) { - k := make([]string, n) - for i := 0; i < len(k); i++ { - k[i] = strconv.Itoa(i) - } - b.ResetTimer() - a := make(map[string]int) - for i := 0; i < b.N; i++ { - a[k[i&(n-1)]] = i - } -} - -func benchmarkMapOperatorAssignStr(b *testing.B, n int) { - k := make([]string, n) - for i := 0; i < len(k); i++ { - k[i] = strconv.Itoa(i) - } - b.ResetTimer() - a := make(map[string]string) - for i := 0; i < b.N; i++ { - key := k[i&(n-1)] - a[key] += key - } -} - -func benchmarkMapAppendAssignStr(b *testing.B, n int) { - k := make([]string, n) - for i := 0; i < len(k); i++ { - k[i] = strconv.Itoa(i) - } - a := make(map[string][]string) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := k[i&(n-1)] - a[key] = append(a[key], key) - } -} - -func benchmarkMapDeleteStr(b *testing.B, n int) { - i2s := make([]string, n) - for i := 0; i < n; i++ { - i2s[i] = strconv.Itoa(i) - } - a := make(map[string]int, n) - b.ResetTimer() - k := 0 - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := 0; j < n; j++ { - a[i2s[j]] = j - } - k = i - b.StartTimer() - } - delete(a, i2s[i-k]) - } -} - -func benchmarkMapDeletePointer(b *testing.B, n int) { - i2p := make([]*int, n) - for i := 0; i < n; i++ { - i2p[i] = new(int) - } - a := make(map[*int]int, n) - b.ResetTimer() - k := 0 - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := 0; j < n; j++ { - a[i2p[j]] = j - } - k = i - b.StartTimer() - } - delete(a, i2p[i-k]) - } -} - -func runWith(f func(*testing.B, int), v ...int) func(*testing.B) { - return func(b *testing.B) { - for _, n := range v { - b.Run(strconv.Itoa(n), func(b *testing.B) { f(b, n) }) - } - } -} - -func BenchmarkMapAssign(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapAssignInt32, 1<<8, 1<<16)) - b.Run("Int64", runWith(benchmarkMapAssignInt64, 1<<8, 1<<16)) - b.Run("Str", runWith(benchmarkMapAssignStr, 1<<8, 1<<16)) -} - -func BenchmarkMapOperatorAssign(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapOperatorAssignInt32, 1<<8, 1<<16)) - b.Run("Int64", runWith(benchmarkMapOperatorAssignInt64, 1<<8, 1<<16)) - b.Run("Str", runWith(benchmarkMapOperatorAssignStr, 1<<8, 1<<16)) -} - -func BenchmarkMapAppendAssign(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapAppendAssignInt32, 1<<8, 1<<16)) - b.Run("Int64", runWith(benchmarkMapAppendAssignInt64, 1<<8, 1<<16)) - b.Run("Str", runWith(benchmarkMapAppendAssignStr, 1<<8, 1<<16)) -} - -func BenchmarkMapDelete(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapDeleteInt32, 100, 1000, 10000)) - b.Run("Int64", runWith(benchmarkMapDeleteInt64, 100, 1000, 10000)) - b.Run("Str", runWith(benchmarkMapDeleteStr, 100, 1000, 10000)) - b.Run("Pointer", runWith(benchmarkMapDeletePointer, 100, 1000, 10000)) -} - func TestDeferDeleteSlow(t *testing.T) { ks := []complex128{0, 1, 2, 3} @@ -1422,22 +1060,11 @@ func TestEmptyMapWithInterfaceKey(t *testing.T) { }) } -func TestLoadFactor(t *testing.T) { - for b := uint8(0); b < 20; b++ { - count := 13 * (1 << b) / 2 // 6.5 - if b == 0 { - count = 8 - } - if runtime.OverLoadFactor(count, b) { - t.Errorf("OverLoadFactor(%d,%d)=true, want false", count, b) - } - if !runtime.OverLoadFactor(count+1, b) { - t.Errorf("OverLoadFactor(%d,%d)=false, want true", count+1, b) - } - } -} - func TestMapKeys(t *testing.T) { + if goexperiment.SwissMap { + t.Skip("mapkeys not implemented for swissmaps") + } + type key struct { s string pad [128]byte // sizeof(key) > abi.MapMaxKeyBytes @@ -1453,6 +1080,10 @@ func TestMapKeys(t *testing.T) { } func TestMapValues(t *testing.T) { + if goexperiment.SwissMap { + t.Skip("mapvalues not implemented for swissmaps") + } + type val struct { s string pad [128]byte // sizeof(val) > abi.MapMaxElemBytes @@ -1544,3 +1175,30 @@ func TestMemHashGlobalSeed(t *testing.T) { } }) } + +func TestMapIterDeleteReplace(t *testing.T) { + inc := 1 + if testing.Short() { + inc = 100 + } + for i := 0; i < 10000; i += inc { + t.Run(fmt.Sprint(i), func(t *testing.T) { + m := make(map[int]bool) + for j := range i { + m[j] = false + } + + // Delete and replace all entries. + for k := range m { + delete(m, k) + m[k] = true + } + + for k, v := range m { + if !v { + t.Errorf("m[%d] got false want true", k) + } + } + }) + } +} diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index 7dc8a1a5..a582a204 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -17,6 +17,7 @@ import ( "internal/abi" "internal/goarch" "internal/goexperiment" + "internal/runtime/sys" "unsafe" ) @@ -91,19 +92,6 @@ import ( // barriers, which will slow down both the mutator and the GC, we always grey // the ptr object regardless of the slot's color. // -// Another place where we intentionally omit memory barriers is when -// accessing mheap_.arena_used to check if a pointer points into the -// heap. On relaxed memory machines, it's possible for a mutator to -// extend the size of the heap by updating arena_used, allocate an -// object from this new region, and publish a pointer to that object, -// but for tracing running on another processor to observe the pointer -// but use the old value of arena_used. In this case, tracing will not -// mark the object, even though it's reachable. However, the mutator -// is guaranteed to execute a write barrier when it publishes the -// pointer, so it will take care of marking the object. A general -// consequence of this is that the garbage collector may cache the -// value of mheap_.arena_used. (See issue #9984.) -// // // Stack writes: // @@ -224,8 +212,8 @@ func wbMove(typ *_type, dst, src unsafe.Pointer) { //go:linkname reflect_typedmemmove reflect.typedmemmove func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) { if raceenabled { - raceWriteObjectPC(typ, dst, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) - raceReadObjectPC(typ, src, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) + raceWriteObjectPC(typ, dst, sys.GetCallerPC(), abi.FuncPCABIInternal(reflect_typedmemmove)) + raceReadObjectPC(typ, src, sys.GetCallerPC(), abi.FuncPCABIInternal(reflect_typedmemmove)) } if msanenabled { msanwrite(dst, typ.Size_) @@ -243,6 +231,11 @@ func reflectlite_typedmemmove(typ *_type, dst, src unsafe.Pointer) { reflect_typedmemmove(typ, dst, src) } +//go:linkname maps_typedmemmove internal/runtime/maps.typedmemmove +func maps_typedmemmove(typ *_type, dst, src unsafe.Pointer) { + typedmemmove(typ, dst, src) +} + // reflectcallmove is invoked by reflectcall to copy the return values // out of the stack and into the heap, invoking the necessary write // barriers. dst, src, and size describe the return value area to @@ -294,7 +287,7 @@ func typedslicecopy(typ *_type, dstPtr unsafe.Pointer, dstLen int, srcPtr unsafe // assignment operations, it's not instrumented in the calling // code and needs its own instrumentation. if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(slicecopy) racewriterangepc(dstPtr, uintptr(n)*typ.Size_, callerpc, pc) racereadrangepc(srcPtr, uintptr(n)*typ.Size_, callerpc, pc) @@ -375,7 +368,7 @@ func typedmemclr(typ *_type, ptr unsafe.Pointer) { memclrNoHeapPointers(ptr, typ.Size_) } -// reflect_typedslicecopy is meant for package reflect, +// reflect_typedmemclr is meant for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/ugorji/go/codec @@ -388,6 +381,11 @@ func reflect_typedmemclr(typ *_type, ptr unsafe.Pointer) { typedmemclr(typ, ptr) } +//go:linkname maps_typedmemclr internal/runtime/maps.typedmemclr +func maps_typedmemclr(typ *_type, ptr unsafe.Pointer) { + typedmemclr(typ, ptr) +} + //go:linkname reflect_typedmemclrpartial reflect.typedmemclrpartial func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintptr) { if writeBarrier.enabled && typ.Pointers() { diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 689fac10..148b2d78 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -59,7 +59,7 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -197,15 +197,14 @@ func (span *mspan) typePointersOfUnchecked(addr uintptr) typePointers { return typePointers{} } } - gcdata := typ.GCData - return typePointers{elem: addr, addr: addr, mask: readUintptr(gcdata), typ: typ} + gcmask := getGCMask(typ) + return typePointers{elem: addr, addr: addr, mask: readUintptr(gcmask), typ: typ} } // typePointersOfType is like typePointersOf, but assumes addr points to one or more -// contiguous instances of the provided type. The provided type must not be nil and -// it must not have its type metadata encoded as a gcprog. +// contiguous instances of the provided type. The provided type must not be nil. // -// It returns an iterator that tiles typ.GCData starting from addr. It's the caller's +// It returns an iterator that tiles typ's gcmask starting from addr. It's the caller's // responsibility to limit iteration. // // nosplit because its callers are nosplit and require all their callees to be nosplit. @@ -213,15 +212,15 @@ func (span *mspan) typePointersOfUnchecked(addr uintptr) typePointers { //go:nosplit func (span *mspan) typePointersOfType(typ *abi.Type, addr uintptr) typePointers { const doubleCheck = false - if doubleCheck && (typ == nil || typ.Kind_&abi.KindGCProg != 0) { + if doubleCheck && typ == nil { throw("bad type passed to typePointersOfType") } if span.spanclass.noscan() { return typePointers{} } // Since we have the type, pretend we have a header. - gcdata := typ.GCData - return typePointers{elem: addr, addr: addr, mask: readUintptr(gcdata), typ: typ} + gcmask := getGCMask(typ) + return typePointers{elem: addr, addr: addr, mask: readUintptr(gcmask), typ: typ} } // nextFast is the fast path of next. nextFast is written to be inlineable and, @@ -295,7 +294,7 @@ func (tp typePointers) next(limit uintptr) (typePointers, uintptr) { } // Grab more bits and try again. - tp.mask = readUintptr(addb(tp.typ.GCData, (tp.addr-tp.elem)/goarch.PtrSize/8)) + tp.mask = readUintptr(addb(getGCMask(tp.typ), (tp.addr-tp.elem)/goarch.PtrSize/8)) if tp.addr+goarch.PtrSize*ptrBits > limit { bits := (tp.addr + goarch.PtrSize*ptrBits - limit) / goarch.PtrSize tp.mask &^= ((1 << (bits)) - 1) << (ptrBits - bits) @@ -345,7 +344,7 @@ func (tp typePointers) fastForward(n, limit uintptr) typePointers { // Move up to the next element. tp.elem += tp.typ.Size_ tp.addr = tp.elem - tp.mask = readUintptr(tp.typ.GCData) + tp.mask = readUintptr(getGCMask(tp.typ)) // We may have exceeded the limit after this. Bail just like next does. if tp.addr >= limit { @@ -354,7 +353,7 @@ func (tp typePointers) fastForward(n, limit uintptr) typePointers { } else { // Grab the mask, but then clear any bits before the target address and any // bits over the limit. - tp.mask = readUintptr(addb(tp.typ.GCData, (tp.addr-tp.elem)/goarch.PtrSize/8)) + tp.mask = readUintptr(addb(getGCMask(tp.typ), (tp.addr-tp.elem)/goarch.PtrSize/8)) tp.mask &^= (1 << ((target - tp.addr) / goarch.PtrSize)) - 1 } if tp.addr+goarch.PtrSize*ptrBits > limit { @@ -457,7 +456,7 @@ func bulkBarrierPreWrite(dst, src, size uintptr, typ *abi.Type) { } var tp typePointers - if typ != nil && typ.Kind_&abi.KindGCProg == 0 { + if typ != nil { tp = s.typePointersOfType(typ, dst) } else { tp = s.typePointersOf(dst, size) @@ -518,7 +517,7 @@ func bulkBarrierPreWriteSrcOnly(dst, src, size uintptr, typ *abi.Type) { } var tp typePointers - if typ != nil && typ.Kind_&abi.KindGCProg == 0 { + if typ != nil { tp = s.typePointersOfType(typ, dst) } else { tp = s.typePointersOf(dst, size) @@ -535,12 +534,13 @@ func bulkBarrierPreWriteSrcOnly(dst, src, size uintptr, typ *abi.Type) { } // initHeapBits initializes the heap bitmap for a span. -// -// TODO(mknyszek): This should set the heap bits for single pointer -// allocations eagerly to avoid calling heapSetType at allocation time, -// just to write one bit. -func (s *mspan) initHeapBits(forceClear bool) { - if (!s.spanclass.noscan() && heapBitsInSpan(s.elemsize)) || s.isUserArenaChunk { +func (s *mspan) initHeapBits() { + if goarch.PtrSize == 8 && !s.spanclass.noscan() && s.spanclass.sizeclass() == 1 { + b := s.heapBits() + for i := range b { + b[i] = ^uintptr(0) + } + } else if (!s.spanclass.noscan() && heapBitsInSpan(s.elemsize)) || s.isUserArenaChunk { b := s.heapBits() clear(b) } @@ -640,37 +640,50 @@ func (span *mspan) heapBitsSmallForAddr(addr uintptr) uintptr { //go:nosplit func (span *mspan) writeHeapBitsSmall(x, dataSize uintptr, typ *_type) (scanSize uintptr) { // The objects here are always really small, so a single load is sufficient. - src0 := readUintptr(typ.GCData) + src0 := readUintptr(getGCMask(typ)) - // Create repetitions of the bitmap if we have a small array. - bits := span.elemsize / goarch.PtrSize + // Create repetitions of the bitmap if we have a small slice backing store. scanSize = typ.PtrBytes src := src0 - switch typ.Size_ { - case goarch.PtrSize: + if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 - default: + } else { + // N.B. We rely on dataSize being an exact multiple of the type size. + // The alternative is to be defensive and mask out src to the length + // of dataSize. The purpose is to save on one additional masking operation. + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ } + if asanenabled { + // Mask src down to dataSize. dataSize is going to be a strange size because of + // the redzone required for allocations when asan is enabled. + src &= (1 << (dataSize / goarch.PtrSize)) - 1 + } } // Since we're never writing more than one uintptr's worth of bits, we're either going // to do one or two writes. - dst := span.heapBits() + dst := unsafe.Pointer(span.base() + pageSize - pageSize/goarch.PtrSize/8) o := (x - span.base()) / goarch.PtrSize i := o / ptrBits j := o % ptrBits + bits := span.elemsize / goarch.PtrSize if j+bits > ptrBits { // Two writes. bits0 := ptrBits - j bits1 := bits - bits0 - dst[i+0] = dst[i+0]&(^uintptr(0)>>bits0) | (src << j) - dst[i+1] = dst[i+1]&^((1<> bits0) + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) } else { // One write. - dst[i] = (dst[i] &^ (((1 << bits) - 1) << j)) | (src << j) + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1< x+maxIterBytes { - size = x + maxIterBytes - interior - } - doubleCheckHeapPointersInterior(x, interior, size, dataSize, gctyp, header, span) + off := alignUp(uintptr(cheaprand())%dataSize, goarch.PtrSize) + size := dataSize - off + if size == 0 { + off -= goarch.PtrSize + size += goarch.PtrSize } - return + interior := x + off + size -= alignDown(uintptr(cheaprand())%size, goarch.PtrSize) + if size == 0 { + size = goarch.PtrSize + } + // Round up the type to the size of the type. + size = (size + gctyp.Size_ - 1) / gctyp.Size_ * gctyp.Size_ + if interior+size > x+maxIterBytes { + size = x + maxIterBytes - interior + } + doubleCheckHeapPointersInterior(x, interior, size, dataSize, gctyp, header, span) } func doubleCheckHeapPointers(x, dataSize uintptr, typ *_type, header **_type, span *mspan) { @@ -794,7 +791,7 @@ func doubleCheckHeapPointers(x, dataSize uintptr, typ *_type, header **_type, sp off := i % typ.Size_ if off < typ.PtrBytes { j := off / goarch.PtrSize - want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0 + want = *addb(getGCMask(typ), j/8)>>(j%8)&1 != 0 } } if want { @@ -817,7 +814,7 @@ func doubleCheckHeapPointers(x, dataSize uintptr, typ *_type, header **_type, sp } println("runtime: extra pointer:", hex(addr)) } - print("runtime: hasHeader=", header != nil, " typ.Size_=", typ.Size_, " hasGCProg=", typ.Kind_&abi.KindGCProg != 0, "\n") + print("runtime: hasHeader=", header != nil, " typ.Size_=", typ.Size_, " TFlagGCMaskOnDemaind=", typ.TFlag&abi.TFlagGCMaskOnDemand != 0, "\n") print("runtime: x=", hex(x), " dataSize=", dataSize, " elemsize=", span.elemsize, "\n") print("runtime: typ=", unsafe.Pointer(typ), " typ.PtrBytes=", typ.PtrBytes, "\n") print("runtime: limit=", hex(x+span.elemsize), "\n") @@ -851,7 +848,7 @@ func doubleCheckHeapPointersInterior(x, interior, size, dataSize uintptr, typ *_ off := i % typ.Size_ if off < typ.PtrBytes { j := off / goarch.PtrSize - want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0 + want = *addb(getGCMask(typ), j/8)>>(j%8)&1 != 0 } } if want { @@ -899,7 +896,7 @@ func doubleCheckHeapPointersInterior(x, interior, size, dataSize uintptr, typ *_ off := i % typ.Size_ if off < typ.PtrBytes { j := off / goarch.PtrSize - want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0 + want = *addb(getGCMask(typ), j/8)>>(j%8)&1 != 0 } } if want { @@ -915,7 +912,7 @@ func doubleCheckHeapPointersInterior(x, interior, size, dataSize uintptr, typ *_ //go:nosplit func doubleCheckTypePointersOfType(s *mspan, typ *_type, addr, size uintptr) { - if typ == nil || typ.Kind_&abi.KindGCProg != 0 { + if typ == nil { return } if typ.Kind_&abi.KindMask == abi.Interface { @@ -1365,9 +1362,6 @@ func bulkBarrierBitmap(dst, src, size, maskOffset uintptr, bits *uint8) { // // The type typ must correspond exactly to [src, src+size) and [dst, dst+size). // dst, src, and size must be pointer-aligned. -// The type typ must have a plain bitmap, not a GC program. -// The only use of this function is in channel sends, and the -// 64 kB channel element limit takes care of this for us. // // Must not be preempted because it typically runs right before memmove, // and the GC must observe them as an atomic action. @@ -1383,14 +1377,10 @@ func typeBitsBulkBarrier(typ *_type, dst, src, size uintptr) { println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " of size ", typ.Size_, " but memory size", size) throw("runtime: invalid typeBitsBulkBarrier") } - if typ.Kind_&abi.KindGCProg != 0 { - println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " with GC prog") - throw("runtime: invalid typeBitsBulkBarrier") - } if !writeBarrier.enabled { return } - ptrmask := typ.GCData + ptrmask := getGCMask(typ) buf := &getg().m.p.ptr().wbBuf var bits uint32 for i := uintptr(0); i < typ.PtrBytes; i += goarch.PtrSize { @@ -1475,6 +1465,9 @@ func progToPointerMask(prog *byte, size uintptr) bitvector { // 0nnnnnnn: emit n bits copied from the next (n+7)/8 bytes // 10000000 n c: repeat the previous n bits c times; n, c are varints // 1nnnnnnn c: repeat the previous n bits c times; c is a varint +// +// Currently, gc programs are only used for describing data and bss +// sections of the binary. // runGCProg returns the number of 1-bit entries written to memory. func runGCProg(prog, dst *byte) uintptr { @@ -1671,24 +1664,6 @@ Run: return totalBits } -// materializeGCProg allocates space for the (1-bit) pointer bitmask -// for an object of size ptrdata. Then it fills that space with the -// pointer bitmask specified by the program prog. -// The bitmask starts at s.startAddr. -// The result must be deallocated with dematerializeGCProg. -func materializeGCProg(ptrdata uintptr, prog *byte) *mspan { - // Each word of ptrdata needs one bit in the bitmap. - bitmapBytes := divRoundUp(ptrdata, 8*goarch.PtrSize) - // Compute the number of pages needed for bitmapBytes. - pages := divRoundUp(bitmapBytes, pageSize) - s := mheap_.allocManual(pages, spanAllocPtrScalarBits) - runGCProg(addb(prog, 4), (*byte)(unsafe.Pointer(s.startAddr))) - return s -} -func dematerializeGCProg(s *mspan) { - mheap_.freeManual(s, spanAllocPtrScalarBits) -} - func dumpGCProg(p *byte) { nptr := 0 for { @@ -1741,13 +1716,13 @@ func dumpGCProg(p *byte) { // //go:linkname reflect_gcbits reflect.gcbits func reflect_gcbits(x any) []byte { - return getgcmask(x) + return pointerMask(x) } // Returns GC type info for the pointer stored in ep for testing. // If ep points to the stack, only static live information will be returned // (i.e. not for objects which are only dynamically live stack objects). -func getgcmask(ep any) (mask []byte) { +func pointerMask(ep any) (mask []byte) { e := *efaceOf(&ep) p := e.data t := e._type @@ -1823,50 +1798,48 @@ func getgcmask(ep any) (mask []byte) { maskFromHeap = maskFromHeap[:len(maskFromHeap)-1] } - if et.Kind_&abi.KindGCProg == 0 { - // Unroll again, but this time from the type information. - maskFromType := make([]byte, (limit-base)/goarch.PtrSize) - tp = s.typePointersOfType(et, base) - for { - var addr uintptr - if tp, addr = tp.next(limit); addr == 0 { - break - } - maskFromType[(addr-base)/goarch.PtrSize] = 1 + // Unroll again, but this time from the type information. + maskFromType := make([]byte, (limit-base)/goarch.PtrSize) + tp = s.typePointersOfType(et, base) + for { + var addr uintptr + if tp, addr = tp.next(limit); addr == 0 { + break } + maskFromType[(addr-base)/goarch.PtrSize] = 1 + } - // Validate that the prefix of maskFromType is equal to - // maskFromHeap. maskFromType may contain more pointers than - // maskFromHeap produces because maskFromHeap may be able to - // get exact type information for certain classes of objects. - // With maskFromType, we're always just tiling the type bitmap - // through to the elemsize. - // - // It's OK if maskFromType has pointers in elemsize that extend - // past the actual populated space; we checked above that all - // that space is zeroed, so just the GC will just see nil pointers. - differs := false - for i := range maskFromHeap { - if maskFromHeap[i] != maskFromType[i] { - differs = true - break - } + // Validate that the prefix of maskFromType is equal to + // maskFromHeap. maskFromType may contain more pointers than + // maskFromHeap produces because maskFromHeap may be able to + // get exact type information for certain classes of objects. + // With maskFromType, we're always just tiling the type bitmap + // through to the elemsize. + // + // It's OK if maskFromType has pointers in elemsize that extend + // past the actual populated space; we checked above that all + // that space is zeroed, so just the GC will just see nil pointers. + differs := false + for i := range maskFromHeap { + if maskFromHeap[i] != maskFromType[i] { + differs = true + break } + } - if differs { - print("runtime: heap mask=") - for _, b := range maskFromHeap { - print(b) - } - println() - print("runtime: type mask=") - for _, b := range maskFromType { - print(b) - } - println() - print("runtime: type=", toRType(et).string(), "\n") - throw("found two different masks from two different methods") + if differs { + print("runtime: heap mask=") + for _, b := range maskFromHeap { + print(b) } + println() + print("runtime: type mask=") + for _, b := range maskFromType { + print(b) + } + println() + print("runtime: type=", toRType(et).string(), "\n") + throw("found two different masks from two different methods") } // Select the heap mask to return. We may not have a type mask. diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index e8da133a..44d737b1 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -6,7 +6,7 @@ package runtime import ( "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -21,8 +21,9 @@ type mcache struct { // The following members are accessed on every malloc, // so they are grouped here for better caching. - nextSample uintptr // trigger heap sample after allocating this many bytes - scanAlloc uintptr // bytes of scannable heap allocated + nextSample int64 // trigger heap sample after allocating this many bytes + memProfRate int // cached mem profile rate, used to detect changes + scanAlloc uintptr // bytes of scannable heap allocated // Allocator cache for tiny objects w/o pointers. // See "Tiny allocator" comment in malloc.go. @@ -252,7 +253,7 @@ func (c *mcache) allocLarge(size uintptr, noscan bool) *mspan { // visible to the background sweeper. mheap_.central[spc].mcentral.fullSwept(mheap_.sweepgen).push(s) s.limit = s.base() + size - s.initHeapBits(false) + s.initHeapBits() return s } diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go index bf597e19..08ff0a5c 100644 --- a/src/runtime/mcentral.go +++ b/src/runtime/mcentral.go @@ -14,7 +14,7 @@ package runtime import ( "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" ) // Central list of free objects of a given size. @@ -260,6 +260,6 @@ func (c *mcentral) grow() *mspan { // n := (npages << _PageShift) / size n := s.divideByElemSize(npages << _PageShift) s.limit = s.base() + size*n - s.initHeapBits(false) + s.initHeapBits() return s } diff --git a/src/runtime/mcheckmark.go b/src/runtime/mcheckmark.go index 258f8892..f5560cf5 100644 --- a/src/runtime/mcheckmark.go +++ b/src/runtime/mcheckmark.go @@ -15,7 +15,7 @@ package runtime import ( "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/mcleanup.go b/src/runtime/mcleanup.go new file mode 100644 index 00000000..972532d4 --- /dev/null +++ b/src/runtime/mcleanup.go @@ -0,0 +1,191 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "internal/abi" + "unsafe" +) + +// AddCleanup attaches a cleanup function to ptr. Some time after ptr is no longer +// reachable, the runtime will call cleanup(arg) in a separate goroutine. +// +// A typical use is that ptr is an object wrapping an underlying resource (e.g., +// a File object wrapping an OS file descriptor), arg is the underlying resource +// (e.g., the OS file descriptor), and the cleanup function releases the underlying +// resource (e.g., by calling the close system call). +// +// There are few constraints on ptr. In particular, multiple cleanups may be +// attached to the same pointer, or to different pointers within the same +// allocation. +// +// If ptr is reachable from cleanup or arg, ptr will never be collected +// and the cleanup will never run. As a protection against simple cases of this, +// AddCleanup panics if arg is equal to ptr. +// +// There is no specified order in which cleanups will run. +// In particular, if several objects point to each other and all become +// unreachable at the same time, their cleanups all become eligible to run +// and can run in any order. This is true even if the objects form a cycle. +// +// A single goroutine runs all cleanup calls for a program, sequentially. If a +// cleanup function must run for a long time, it should create a new goroutine. +// +// If ptr has both a cleanup and a finalizer, the cleanup will only run once +// it has been finalized and becomes unreachable without an associated finalizer. +// +// The cleanup(arg) call is not always guaranteed to run; in particular it is not +// guaranteed to run before program exit. +// +// Cleanups are not guaranteed to run if the size of T is zero bytes, because +// it may share same address with other zero-size objects in memory. See +// https://go.dev/ref/spec#Size_and_alignment_guarantees. +// +// It is not guaranteed that a cleanup will run for objects allocated +// in initializers for package-level variables. Such objects may be +// linker-allocated, not heap-allocated. +// +// Note that because cleanups may execute arbitrarily far into the future +// after an object is no longer referenced, the runtime is allowed to perform +// a space-saving optimization that batches objects together in a single +// allocation slot. The cleanup for an unreferenced object in such an +// allocation may never run if it always exists in the same batch as a +// referenced object. Typically, this batching only happens for tiny +// (on the order of 16 bytes or less) and pointer-free objects. +// +// A cleanup may run as soon as an object becomes unreachable. +// In order to use cleanups correctly, the program must ensure that +// the object is reachable until it is safe to run its cleanup. +// Objects stored in global variables, or that can be found by tracing +// pointers from a global variable, are reachable. A function argument or +// receiver may become unreachable at the last point where the function +// mentions it. To ensure a cleanup does not get called prematurely, +// pass the object to the [KeepAlive] function after the last point +// where the object must remain reachable. +func AddCleanup[T, S any](ptr *T, cleanup func(S), arg S) Cleanup { + // Explicitly force ptr to escape to the heap. + ptr = abi.Escape(ptr) + + // The pointer to the object must be valid. + if ptr == nil { + panic("runtime.AddCleanup: ptr is nil") + } + usptr := uintptr(unsafe.Pointer(ptr)) + + // Check that arg is not equal to ptr. + if kind := abi.TypeOf(arg).Kind(); kind == abi.Pointer || kind == abi.UnsafePointer { + if unsafe.Pointer(ptr) == *((*unsafe.Pointer)(unsafe.Pointer(&arg))) { + panic("runtime.AddCleanup: ptr is equal to arg, cleanup will never run") + } + } + if inUserArenaChunk(usptr) { + // Arena-allocated objects are not eligible for cleanup. + panic("runtime.AddCleanup: ptr is arena-allocated") + } + if debug.sbrk != 0 { + // debug.sbrk never frees memory, so no cleanup will ever run + // (and we don't have the data structures to record them). + // Return a noop cleanup. + return Cleanup{} + } + + fn := func() { + cleanup(arg) + } + // Closure must escape. + fv := *(**funcval)(unsafe.Pointer(&fn)) + fv = abi.Escape(fv) + + // Find the containing object. + base, _, _ := findObject(usptr, 0, 0) + if base == 0 { + if isGoPointerWithoutSpan(unsafe.Pointer(ptr)) { + // Cleanup is a noop. + return Cleanup{} + } + panic("runtime.AddCleanup: ptr not in allocated block") + } + + // Ensure we have a finalizer processing goroutine running. + createfing() + + id := addCleanup(unsafe.Pointer(ptr), fv) + return Cleanup{ + id: id, + ptr: usptr, + } +} + +// Cleanup is a handle to a cleanup call for a specific object. +type Cleanup struct { + // id is the unique identifier for the cleanup within the arena. + id uint64 + // ptr contains the pointer to the object. + ptr uintptr +} + +// Stop cancels the cleanup call. Stop will have no effect if the cleanup call +// has already been queued for execution (because ptr became unreachable). +// To guarantee that Stop removes the cleanup function, the caller must ensure +// that the pointer that was passed to AddCleanup is reachable across the call to Stop. +func (c Cleanup) Stop() { + if c.id == 0 { + // id is set to zero when the cleanup is a noop. + return + } + + // The following block removes the Special record of type cleanup for the object c.ptr. + span := spanOfHeap(uintptr(unsafe.Pointer(c.ptr))) + if span == nil { + return + } + // Ensure that the span is swept. + // Sweeping accesses the specials list w/o locks, so we have + // to synchronize with it. And it's just much safer. + mp := acquirem() + span.ensureSwept() + + offset := uintptr(unsafe.Pointer(c.ptr)) - span.base() + + var found *special + lock(&span.speciallock) + + iter, exists := span.specialFindSplicePoint(offset, _KindSpecialCleanup) + if exists { + for { + s := *iter + if s == nil { + // Reached the end of the linked list. Stop searching at this point. + break + } + if offset == uintptr(s.offset) && _KindSpecialCleanup == s.kind && + (*specialCleanup)(unsafe.Pointer(s)).id == c.id { + // The special is a cleanup and contains a matching cleanup id. + *iter = s.next + found = s + break + } + if offset < uintptr(s.offset) || (offset == uintptr(s.offset) && _KindSpecialCleanup < s.kind) { + // The special is outside the region specified for that kind of + // special. The specials are sorted by kind. + break + } + // Try the next special. + iter = &s.next + } + } + if span.specials == nil { + spanHasNoSpecials(span) + } + unlock(&span.speciallock) + releasem(mp) + + if found == nil { + return + } + lock(&mheap_.speciallock) + mheap_.specialCleanupAlloc.free(unsafe.Pointer(found)) + unlock(&mheap_.speciallock) +} diff --git a/src/runtime/mcleanup_test.go b/src/runtime/mcleanup_test.go new file mode 100644 index 00000000..d62356fe --- /dev/null +++ b/src/runtime/mcleanup_test.go @@ -0,0 +1,298 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "runtime" + "testing" + "unsafe" +) + +func TestCleanup(t *testing.T) { + ch := make(chan bool, 1) + done := make(chan bool, 1) + want := 97531 + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- true + } + runtime.AddCleanup(v, cleanup, 97531) + v = nil + done <- true + }() + <-done + runtime.GC() + <-ch +} + +func TestCleanupMultiple(t *testing.T) { + ch := make(chan bool, 3) + done := make(chan bool, 1) + want := 97531 + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- true + } + runtime.AddCleanup(v, cleanup, 97531) + runtime.AddCleanup(v, cleanup, 97531) + runtime.AddCleanup(v, cleanup, 97531) + v = nil + done <- true + }() + <-done + runtime.GC() + <-ch + <-ch + <-ch +} + +func TestCleanupZeroSizedStruct(t *testing.T) { + type Z struct{} + z := new(Z) + runtime.AddCleanup(z, func(s string) {}, "foo") +} + +func TestCleanupAfterFinalizer(t *testing.T) { + ch := make(chan int, 2) + done := make(chan bool, 1) + want := 97531 + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + finalizer := func(x *int) { + ch <- 1 + } + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- 2 + } + runtime.AddCleanup(v, cleanup, 97531) + runtime.SetFinalizer(v, finalizer) + v = nil + done <- true + }() + <-done + runtime.GC() + var result int + result = <-ch + if result != 1 { + t.Errorf("result %d, want 1", result) + } + runtime.GC() + result = <-ch + if result != 2 { + t.Errorf("result %d, want 2", result) + } +} + +func TestCleanupInteriorPointer(t *testing.T) { + ch := make(chan bool, 3) + done := make(chan bool, 1) + want := 97531 + go func() { + // Allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + p unsafe.Pointer + i int + a int + b int + c int + } + ts := new(T) + ts.a = 97531 + ts.b = 97531 + ts.c = 97531 + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- true + } + runtime.AddCleanup(&ts.a, cleanup, 97531) + runtime.AddCleanup(&ts.b, cleanup, 97531) + runtime.AddCleanup(&ts.c, cleanup, 97531) + ts = nil + done <- true + }() + <-done + runtime.GC() + <-ch + <-ch + <-ch +} + +func TestCleanupStop(t *testing.T) { + done := make(chan bool, 1) + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + t.Error("cleanup called, want no cleanup called") + } + c := runtime.AddCleanup(v, cleanup, 97531) + c.Stop() + v = nil + done <- true + }() + <-done + runtime.GC() +} + +func TestCleanupStopMultiple(t *testing.T) { + done := make(chan bool, 1) + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + t.Error("cleanup called, want no cleanup called") + } + c := runtime.AddCleanup(v, cleanup, 97531) + c.Stop() + c.Stop() + c.Stop() + v = nil + done <- true + }() + <-done + runtime.GC() +} + +func TestCleanupStopinterleavedMultiple(t *testing.T) { + ch := make(chan bool, 3) + done := make(chan bool, 1) + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + if x != 1 { + t.Error("cleanup called, want no cleanup called") + } + ch <- true + } + runtime.AddCleanup(v, cleanup, 1) + runtime.AddCleanup(v, cleanup, 2).Stop() + runtime.AddCleanup(v, cleanup, 1) + runtime.AddCleanup(v, cleanup, 2).Stop() + runtime.AddCleanup(v, cleanup, 1) + v = nil + done <- true + }() + <-done + runtime.GC() + <-ch + <-ch + <-ch +} + +func TestCleanupStopAfterCleanupRuns(t *testing.T) { + ch := make(chan bool, 1) + done := make(chan bool, 1) + var stop func() + go func() { + // Allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + ch <- true + } + cl := runtime.AddCleanup(v, cleanup, 97531) + v = nil + stop = cl.Stop + done <- true + }() + <-done + runtime.GC() + <-ch + stop() +} + +func TestCleanupPointerEqualsArg(t *testing.T) { + // See go.dev/issue/71316 + defer func() { + want := "runtime.AddCleanup: ptr is equal to arg, cleanup will never run" + if r := recover(); r == nil { + t.Error("want panic, test did not panic") + } else if r == want { + // do nothing + } else { + t.Errorf("wrong panic: want=%q, got=%q", want, r) + } + }() + + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + runtime.AddCleanup(v, func(x *int) {}, v) + v = nil + runtime.GC() +} diff --git a/src/runtime/mem_nonsbrk.go b/src/runtime/mem_nonsbrk.go new file mode 100644 index 00000000..41b7260e --- /dev/null +++ b/src/runtime/mem_nonsbrk.go @@ -0,0 +1,15 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !plan9 && !wasm + +package runtime + +import "unsafe" + +const isSbrkPlatform = false + +func sysReserveAlignedSbrk(size, align uintptr) (unsafe.Pointer, uintptr) { + panic("unreachable") +} diff --git a/src/runtime/mem_sbrk.go b/src/runtime/mem_sbrk.go index dc0a764a..cfca8910 100644 --- a/src/runtime/mem_sbrk.go +++ b/src/runtime/mem_sbrk.go @@ -8,10 +8,32 @@ package runtime import "unsafe" +const isSbrkPlatform = true + const memDebug = false -var bloc uintptr -var blocMax uintptr +// Memory management on sbrk systems (including the linear memory +// on Wasm). + +// bloc is the runtime's sense of the break, which can go up or +// down. blocMax is the system's break, also the high water mark +// of bloc. The runtime uses memory up to bloc. The memory +// between bloc and blocMax is allocated by the OS but not used +// by the runtime. +// +// When the runtime needs to grow the heap address range, it +// increases bloc. When it needs to grow beyond blocMax, it calls +// the system sbrk to allocate more memory (and therefore +// increase blocMax). +// +// When the runtime frees memory at the end of the address space, +// it decreases bloc, but does not reduces the system break (as +// the OS doesn't support it). When the runtime frees memory in +// the middle of the address space, the memory goes to a free +// list. + +var bloc uintptr // The runtime's sense of break. Can go up or down. +var blocMax uintptr // The break of the OS. Only increase. var memlock mutex type memHdr struct { @@ -27,6 +49,13 @@ func (p memHdrPtr) ptr() *memHdr { return (*memHdr)(unsafe.Pointer(p)) } func (p *memHdrPtr) set(x *memHdr) { *p = memHdrPtr(unsafe.Pointer(x)) } func memAlloc(n uintptr) unsafe.Pointer { + if p := memAllocNoGrow(n); p != nil { + return p + } + return sbrk(n) +} + +func memAllocNoGrow(n uintptr) unsafe.Pointer { n = memRound(n) var prevp *memHdr for p := memFreelist.ptr(); p != nil; p = p.next.ptr() { @@ -46,7 +75,7 @@ func memAlloc(n uintptr) unsafe.Pointer { } prevp = p } - return sbrk(n) + return nil } func memFree(ap unsafe.Pointer, n uintptr) { @@ -187,3 +216,34 @@ func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { unlock(&memlock) return p } + +func sysReserveAlignedSbrk(size, align uintptr) (unsafe.Pointer, uintptr) { + lock(&memlock) + if p := memAllocNoGrow(size + align); p != nil { + // We can satisfy the reservation from the free list. + // Trim off the unaligned parts. + pAligned := alignUp(uintptr(p), align) + if startLen := pAligned - uintptr(p); startLen > 0 { + memFree(p, startLen) + } + end := pAligned + size + if endLen := (uintptr(p) + size + align) - end; endLen > 0 { + memFree(unsafe.Pointer(end), endLen) + } + memCheck() + return unsafe.Pointer(pAligned), size + } + + // Round up bloc to align, then allocate size. + p := alignUp(bloc, align) + r := sbrk(p + size - bloc) + if r == nil { + p, size = 0, 0 + } else if l := p - uintptr(r); l > 0 { + // Free the area we skipped over for alignment. + memFree(r, l) + memCheck() + } + unlock(&memlock) + return unsafe.Pointer(p), size +} diff --git a/src/runtime/mem_wasm.go b/src/runtime/mem_wasm.go index d9d32705..76de88ac 100644 --- a/src/runtime/mem_wasm.go +++ b/src/runtime/mem_wasm.go @@ -7,14 +7,21 @@ package runtime import "unsafe" func sbrk(n uintptr) unsafe.Pointer { - grow := divRoundUp(n, physPageSize) - size := growMemory(int32(grow)) - if size < 0 { - return nil + bl := bloc + n = memRound(n) + if bl+n > blocMax { + grow := (bl + n - blocMax) / physPageSize + size := growMemory(int32(grow)) + if size < 0 { + return nil + } + resetMemoryDataView() + blocMax = bl + n } - resetMemoryDataView() - return unsafe.Pointer(uintptr(size) * physPageSize) + bloc += n + return unsafe.Pointer(bl) } // Implemented in src/runtime/sys_wasm.s func growMemory(pages int32) int32 +func currentMemory() int32 diff --git a/src/runtime/memclr_arm64.s b/src/runtime/memclr_arm64.s index 1c35dfe0..3e49f7fc 100644 --- a/src/runtime/memclr_arm64.s +++ b/src/runtime/memclr_arm64.s @@ -82,6 +82,7 @@ last16: last_end: RET + PCALIGN $16 no_zva: SUB $16, R0, R0 SUB $64, R1, R1 @@ -98,6 +99,7 @@ loop_64: BNE tail63 RET + PCALIGN $16 try_zva: // Try using the ZVA feature to zero entire cache lines // It is not meaningful to use ZVA if the block size is less than 64, @@ -124,6 +126,7 @@ try_zva: MOVW R5, block_size<>(SB) B no_zva + PCALIGN $16 init: MOVW $4, R9 ANDW $15, R3, R5 @@ -134,6 +137,7 @@ init: // Block size is less than 64. BNE no_zva + PCALIGN $16 zero_by_line: CMP R5, R1 // Not enough memory to reach alignment @@ -170,6 +174,7 @@ loop_zva_prolog: aligned: SUB R5, R1, R1 + PCALIGN $16 loop_zva: WORD $0xd50b7420 // DC ZVA, R0 ADD R5, R0, R0 diff --git a/src/runtime/memclr_loong64.s b/src/runtime/memclr_loong64.s index 1d45e82d..346b210c 100644 --- a/src/runtime/memclr_loong64.s +++ b/src/runtime/memclr_loong64.s @@ -5,36 +5,131 @@ #include "go_asm.h" #include "textflag.h" +// Register map +// +// R4: ptr +// R5: n +// R6: ptrend +// R7: tmp + +// Algorithm: +// +// 1. when count <= 64 bytes, memory alignment check is omitted. +// The handling is divided into distinct cases based on the size +// of count: clr_0, clr_1, clr_2, clr_3, clr_4, clr_5through7, +// clr_8, clr_9through16, clr_17through32, and clr_33through64. +// +// 2. when count > 64 bytes, memory alignment check is performed. +// Unaligned bytes are processed first (that is, 8-(ptr&7)), and +// then a 64-byte loop is executed to zero out memory. +// When the number of remaining bytes not cleared is n < 64 bytes, +// a tail processing is performed, invoking the corresponding case +// based on the size of n. +// +// ptr newptr ptrend +// | |<----count after correction---->| +// |<-------------count before correction---------->| +// |<--8-(ptr&7)-->| |<---64 bytes--->| +// +------------------------------------------------+ +// | Head | Body | Tail | +// +---------------+---------------+----------------+ +// newptr = ptr - (ptr & 7) + 8 +// count = count - 8 + (ptr & 7) + // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16 + BEQ R5, clr_0 ADDV R4, R5, R6 - // if less than 8 bytes, do one byte at a time - SGTU $8, R5, R8 - BNE R8, out +tail: + // <=64 bytes, clear directly, not check aligned + SGTU $2, R5, R7 + BNE R7, clr_1 + SGTU $3, R5, R7 + BNE R7, clr_2 + SGTU $4, R5, R7 + BNE R7, clr_3 + SGTU $5, R5, R7 + BNE R7, clr_4 + SGTU $8, R5, R7 + BNE R7, clr_5through7 + SGTU $9, R5, R7 + BNE R7, clr_8 + SGTU $17, R5, R7 + BNE R7, clr_9through16 + SGTU $33, R5, R7 + BNE R7, clr_17through32 + SGTU $65, R5, R7 + BNE R7, clr_33through64 - // do one byte at a time until 8-aligned - AND $7, R4, R8 - BEQ R8, words - MOVB R0, (R4) - ADDV $1, R4 - JMP -4(PC) + // n > 64 bytes, check aligned + AND $7, R4, R7 + BEQ R7, body -words: - // do 8 bytes at a time if there is room - ADDV $-7, R6, R5 - - PCALIGN $16 - SGTU R5, R4, R8 - BEQ R8, out +head: MOVV R0, (R4) - ADDV $8, R4 - JMP -4(PC) + SUBV R7, R4 + ADDV R7, R5 + ADDV $8, R4 // newptr = ptr + (8 - (ptr & 7)) + SUBV $8, R5 // newn = n - (8 - (ptr & 7)) + SGTU $65, R5, R7 + BNE R7, clr_33through64 -out: - BEQ R4, R6, done - MOVB R0, (R4) - ADDV $1, R4 - JMP -3(PC) -done: +body: + MOVV R0, (R4) + MOVV R0, 8(R4) + MOVV R0, 16(R4) + MOVV R0, 24(R4) + MOVV R0, 32(R4) + MOVV R0, 40(R4) + MOVV R0, 48(R4) + MOVV R0, 56(R4) + ADDV $-64, R5 + ADDV $64, R4 + SGTU $65, R5, R7 + BEQ R7, body + BEQ R5, clr_0 + JMP tail + +clr_0: + RET +clr_1: + MOVB R0, (R4) + RET +clr_2: + MOVH R0, (R4) + RET +clr_3: + MOVH R0, (R4) + MOVB R0, 2(R4) + RET +clr_4: + MOVW R0, (R4) + RET +clr_5through7: + MOVW R0, (R4) + MOVW R0, -4(R6) + RET +clr_8: + MOVV R0, (R4) + RET +clr_9through16: + MOVV R0, (R4) + MOVV R0, -8(R6) + RET +clr_17through32: + MOVV R0, (R4) + MOVV R0, 8(R4) + MOVV R0, -16(R6) + MOVV R0, -8(R6) + RET +clr_33through64: + MOVV R0, (R4) + MOVV R0, 8(R4) + MOVV R0, 16(R4) + MOVV R0, 24(R4) + MOVV R0, -32(R6) + MOVV R0, -24(R6) + MOVV R0, -16(R6) + MOVV R0, -8(R6) RET diff --git a/src/runtime/memclr_ppc64x.s b/src/runtime/memclr_ppc64x.s index bc4b3fc2..ffe40e12 100644 --- a/src/runtime/memclr_ppc64x.s +++ b/src/runtime/memclr_ppc64x.s @@ -19,7 +19,7 @@ check: SRD $3, R4, R6 // R6: double words to clear CMP R6, $0, CR1 // CR1[EQ] set if no double words - BC 12, 6, nozerolarge // only single bytes + BEQ CR1, nozerolarge // only single bytes CMP R4, $512 BLT under512 // special case for < 512 ANDCC $127, R3, R8 // check for 128 alignment of address @@ -104,7 +104,7 @@ lt16gt8: #endif nozerolarge: ANDCC $7, R4, R5 // any remaining bytes - BC 4, 1, LR // ble lr + BLE CR0, LR // ble lr #ifdef GOPPC64_power10 XXLXOR VS32, VS32, VS32 // clear VS32 (V0) SLD $56, R5, R7 @@ -124,7 +124,7 @@ next2: ADD $-2, R5 next1: CMP R5, $0 - BC 12, 2, LR // beqlr + BEQ CR0, LR // beqlr MOVB R0, 0(R3) RET #endif diff --git a/src/runtime/memmove_amd64.s b/src/runtime/memmove_amd64.s index 018bb0b1..8883b55e 100644 --- a/src/runtime/memmove_amd64.s +++ b/src/runtime/memmove_amd64.s @@ -72,9 +72,10 @@ tail: CMPQ BX, $256 JBE move_129through256 - TESTB $1, runtime·useAVXmemmove(SB) - JNZ avxUnaligned - + MOVB runtime·memmoveBits(SB), AX + // We have AVX but we don't want to use REP MOVSx. + CMPB AX, $const_avxSupported + JEQ avxUnaligned /* * check and set for backwards */ @@ -82,16 +83,23 @@ tail: JLS back /* - * forward copy loop - */ +* forward copy loop +*/ forward: + CMPQ BX, $2048 + JL check_avx + // REP MOVSx is slow if destination address is unaligned. + TESTQ $15,DI + JNZ check_avx + TESTB $const_repmovsPreferred, AX + JNZ fwdBy8 + // For backward copy, REP MOVSx performs worse than avx. +check_avx: + TESTB $const_avxSupported, AX + JNZ avxUnaligned + CMPQ BX, $2048 JLS move_256through2048 - - // If REP MOVSB isn't fast, don't use it - CMPB internal∕cpu·X86+const_offsetX86HasERMS(SB), $1 // enhanced REP MOVSB/STOSB - JNE fwdBy8 - // Check alignment MOVL SI, AX ORL DI, AX @@ -104,12 +112,16 @@ forward: RET fwdBy8: + // Loading the last (possibly partially overlapping) word and writing + // it at the end. + MOVQ -8(SI)(BX*1), AX + LEAQ -8(DI)(BX*1), DX // Do 8 bytes at a time - MOVQ BX, CX + LEAQ -1(BX),CX SHRQ $3, CX - ANDQ $7, BX REP; MOVSQ - JMP tail + MOVQ AX, (DX) + RET back: /* @@ -119,6 +131,9 @@ back: ADDQ BX, CX CMPQ CX, DI JLS forward + + TESTB $const_avxSupported, AX + JNZ avxUnaligned /* * whole thing backwards has * adjusted addresses diff --git a/src/runtime/memmove_linux_amd64_test.go b/src/runtime/memmove_linux_amd64_test.go index 5f900623..c5588115 100644 --- a/src/runtime/memmove_linux_amd64_test.go +++ b/src/runtime/memmove_linux_amd64_test.go @@ -5,6 +5,7 @@ package runtime_test import ( + "internal/asan" "os" "syscall" "testing" @@ -14,6 +15,10 @@ import ( // TestMemmoveOverflow maps 3GB of memory and calls memmove on // the corresponding slice. func TestMemmoveOverflow(t *testing.T) { + if asan.Enabled { + t.Skip("appears to break asan and causes spurious failures") + } + t.Parallel() // Create a temporary file. tmp, err := os.CreateTemp("", "go-memmovetest") diff --git a/src/runtime/memmove_loong64.s b/src/runtime/memmove_loong64.s index a94cf999..8827ca07 100644 --- a/src/runtime/memmove_loong64.s +++ b/src/runtime/memmove_loong64.s @@ -6,99 +6,266 @@ // See memmove Go doc for important implementation constraints. +// Register map +// +// to R4 +// from R5 +// n(aka count) R6 +// to-end R7 +// from-end R8 +// data R11-R18 +// tmp R9 + +// Algorithm: +// +// Memory alignment check is only performed for copy size greater +// than 64 bytes to minimize overhead. +// +// when copy size <= 64 bytes, jump to label tail, according to the +// copy size to select the appropriate case and copy directly. +// Based on the common memory access instructions of loong64, the +// currently implemented cases are: +// move_0, move_1, move_2, move_3, move_4, move_5through7, move_8, +// move_9through16, move_17through32, move_33through64 +// +// when copy size > 64 bytes, use the destination-aligned copying, +// adopt the following strategy to copy in 3 parts: +// 1. Head: do the memory alignment +// 2. Body: a 64-byte loop structure +// 3. Tail: processing of the remaining part (<= 64 bytes) +// +// forward: +// +// Dst NewDst Dstend +// | |<----count after correction---->| +// |<-------------count before correction---------->| +// |<--8-(Dst&7)-->| |<---64 bytes--->| +// +------------------------------------------------+ +// | Head | Body | Tail | +// +---------------+---------------+----------------+ +// NewDst = Dst - (Dst & 7) + 8 +// count = count - 8 + (Dst & 7) +// Src = Src - (Dst & 7) + 8 +// +// backward: +// +// Dst NewDstend Dstend +// |<-----count after correction------>| | +// |<------------count before correction--------------->| +// |<---64 bytes--->| |<---Dstend&7--->| +// +----------------------------------------------------+ +// | Tail | Body | Head | +// +----------------+------------------+----------------+ +// NewDstend = Dstend - (Dstend & 7) +// count = count - (Dstend & 7) +// Srcend = Srcend - (Dstend & 7) + // func memmove(to, from unsafe.Pointer, n uintptr) TEXT runtime·memmove(SB), NOSPLIT|NOFRAME, $0-24 - BNE R6, check - RET + BEQ R4, R5, move_0 + BEQ R6, move_0 -check: - SGTU R4, R5, R7 - BNE R7, backward + ADDV R4, R6, R7 // to-end pointer + ADDV R5, R6, R8 // from-end pointer - ADDV R4, R6, R9 // end pointer +tail: + //copy size <= 64 bytes, copy directly, not check aligned - // if the two pointers are not of same alignments, do byte copying - SUBVU R5, R4, R7 - AND $7, R7 - BNE R7, out + // < 2 bytes + SGTU $2, R6, R9 + BNE R9, move_1 - // if less than 8 bytes, do byte copying - SGTU $8, R6, R7 - BNE R7, out + // < 3 bytes + SGTU $3, R6, R9 + BNE R9, move_2 - // do one byte at a time until 8-aligned - AND $7, R4, R8 - BEQ R8, words - MOVB (R5), R7 + // < 4 bytes + SGTU $4, R6, R9 + BNE R9, move_3 + + // < 5 bytes + SGTU $5, R6, R9 + BNE R9, move_4 + + // >= 5 bytes and < 8 bytes + SGTU $8, R6, R9 + BNE R9, move_5through7 + + // < 9 bytes + SGTU $9, R6, R9 + BNE R9, move_8 + + // >= 9 bytes and < 17 bytes + SGTU $17, R6, R9 + BNE R9, move_9through16 + + // >= 17 bytes and < 33 bytes + SGTU $33, R6, R9 + BNE R9, move_17through32 + + // >= 33 bytes and < 65 bytes + SGTU $65, R6, R9 + BNE R9, move_33through64 + + // if (dst > src) && (dst < src + count), regarded as memory + // overlap, jump to backward + // else, jump to forward + BGEU R5, R4, forward + ADDV R5, R6, R10 + BLTU R4, R10, backward + +forward: + AND $7, R4, R9 // dst & 7 + BEQ R9, body +head: + MOVV $8, R10 + SUBV R9, R10 // head = 8 - (dst & 7) + MOVB (R5), R11 + SUBV $1, R10 ADDV $1, R5 - MOVB R7, (R4) + MOVB R11, (R4) ADDV $1, R4 - JMP -6(PC) + BNE R10, -5(PC) + ADDV R9, R6 + ADDV $-8, R6 // newcount = count + (dst & 7) - 8 + // if newcount < 65 bytes, use move_33through64 to copy is enough + SGTU $65, R6, R9 + BNE R9, move_33through64 -words: - // do 8 bytes at a time if there is room - ADDV $-7, R9, R6 // R6 is end pointer-7 - - PCALIGN $16 - SGTU R6, R4, R8 - BEQ R8, out - MOVV (R5), R7 - ADDV $8, R5 - MOVV R7, (R4) - ADDV $8, R4 - JMP -6(PC) - -out: - BEQ R4, R9, done - MOVB (R5), R7 - ADDV $1, R5 - MOVB R7, (R4) - ADDV $1, R4 - JMP -5(PC) -done: - RET +body: + MOVV (R5), R11 + MOVV 8(R5), R12 + MOVV 16(R5), R13 + MOVV 24(R5), R14 + MOVV 32(R5), R15 + MOVV 40(R5), R16 + MOVV 48(R5), R17 + MOVV 56(R5), R18 + MOVV R11, (R4) + MOVV R12, 8(R4) + MOVV R13, 16(R4) + MOVV R14, 24(R4) + MOVV R15, 32(R4) + MOVV R16, 40(R4) + MOVV R17, 48(R4) + MOVV R18, 56(R4) + ADDV $-64, R6 + ADDV $64, R4 + ADDV $64, R5 + SGTU $64, R6, R9 + // if the remaining part >= 64 bytes, jmp to body + BEQ R9, body + // if the remaining part == 0 bytes, use move_0 to return + BEQ R6, move_0 + // if the remaining part in (0, 63] bytes, jmp to tail + JMP tail +// The backward copy algorithm is the same as the forward copy, +// except for the direction. backward: - ADDV R6, R5 // from-end pointer - ADDV R4, R6, R9 // to-end pointer + AND $7, R7, R9 // dstend & 7 + BEQ R9, b_body +b_head: + MOVV -8(R8), R11 + SUBV R9, R6 // newcount = count - (dstend & 7) + SUBV R9, R8 // newsrcend = srcend - (dstend & 7) + MOVV -8(R8), R12 + MOVV R11, -8(R7) + SUBV R9, R7 // newdstend = dstend - (dstend & 7) + MOVV R12, -8(R7) + SUBV $8, R6 + SUBV $8, R7 + SUBV $8, R8 + SGTU $65, R6, R9 + BNE R9, move_33through64 - // if the two pointers are not of same alignments, do byte copying - SUBVU R9, R5, R7 - AND $7, R7 - BNE R7, out1 +b_body: + MOVV -8(R8), R11 + MOVV -16(R8), R12 + MOVV -24(R8), R13 + MOVV -32(R8), R14 + MOVV -40(R8), R15 + MOVV -48(R8), R16 + MOVV -56(R8), R17 + MOVV -64(R8), R18 + MOVV R11, -8(R7) + MOVV R12, -16(R7) + MOVV R13, -24(R7) + MOVV R14, -32(R7) + MOVV R15, -40(R7) + MOVV R16, -48(R7) + MOVV R17, -56(R7) + MOVV R18, -64(R7) + ADDV $-64, R6 + ADDV $-64, R7 + ADDV $-64, R8 + SGTU $64, R6, R9 + BEQ R9, b_body + BEQ R6, move_0 + JMP tail - // if less than 8 bytes, do byte copying - SGTU $8, R6, R7 - BNE R7, out1 - - // do one byte at a time until 8-aligned - AND $7, R9, R8 - BEQ R8, words1 - ADDV $-1, R5 - MOVB (R5), R7 - ADDV $-1, R9 - MOVB R7, (R9) - JMP -6(PC) - -words1: - // do 8 bytes at a time if there is room - ADDV $7, R4, R6 // R6 is start pointer+7 - - PCALIGN $16 - SGTU R9, R6, R8 - BEQ R8, out1 - ADDV $-8, R5 - MOVV (R5), R7 - ADDV $-8, R9 - MOVV R7, (R9) - JMP -6(PC) - -out1: - BEQ R4, R9, done1 - ADDV $-1, R5 - MOVB (R5), R7 - ADDV $-1, R9 - MOVB R7, (R9) - JMP -5(PC) -done1: +move_0: + RET + +move_1: + MOVB (R5), R11 + MOVB R11, (R4) + RET +move_2: + MOVH (R5), R11 + MOVH R11, (R4) + RET +move_3: + MOVH (R5), R11 + MOVB -1(R8), R12 + MOVH R11, (R4) + MOVB R12, -1(R7) + RET +move_4: + MOVW (R5), R11 + MOVW R11, (R4) + RET +move_5through7: + MOVW (R5), R11 + MOVW -4(R8), R12 + MOVW R11, (R4) + MOVW R12, -4(R7) + RET +move_8: + MOVV (R5), R11 + MOVV R11, (R4) + RET +move_9through16: + MOVV (R5), R11 + MOVV -8(R8), R12 + MOVV R11, (R4) + MOVV R12, -8(R7) + RET +move_17through32: + MOVV (R5), R11 + MOVV 8(R5), R12 + MOVV -16(R8), R13 + MOVV -8(R8), R14 + MOVV R11, (R4) + MOVV R12, 8(R4) + MOVV R13, -16(R7) + MOVV R14, -8(R7) + RET +move_33through64: + MOVV (R5), R11 + MOVV 8(R5), R12 + MOVV 16(R5), R13 + MOVV 24(R5), R14 + MOVV -32(R8), R15 + MOVV -24(R8), R16 + MOVV -16(R8), R17 + MOVV -8(R8), R18 + MOVV R11, (R4) + MOVV R12, 8(R4) + MOVV R13, 16(R4) + MOVV R14, 24(R4) + MOVV R15, -32(R7) + MOVV R16, -24(R7) + MOVV R17, -16(R7) + MOVV R18, -8(R7) RET diff --git a/src/runtime/memmove_ppc64x.s b/src/runtime/memmove_ppc64x.s index 18b9c850..9892028d 100644 --- a/src/runtime/memmove_ppc64x.s +++ b/src/runtime/memmove_ppc64x.s @@ -60,11 +60,11 @@ mcopy: SUB SRC, TGT, TMP // dest - src CMPU TMP, LEN, CR2 // < len? - BC 12, 8, backward // BLT CR2 backward + BLT CR2, backward // Copying forward if no overlap. - BC 12, 6, checkbytes // BEQ CR1, checkbytes + BEQ CR1, checkbytes SRDCC $3, DWORDS, OCTWORDS // 64 byte chunks? MOVD $16, IDX16 BEQ lt64gt8 // < 64 bytes @@ -132,7 +132,7 @@ lt16: // Move 8 bytes if possible MOVD TMP, 0(TGT) ADD $8, TGT checkbytes: - BC 12, 14, LR // BEQ lr + BEQ CR3, LR #ifdef GOPPC64_power10 SLD $56, BYTES, TMP LXVL SRC, TMP, V0 @@ -157,7 +157,7 @@ lt4: // Move halfword if possible ADD $2, TGT lt2: // Move last byte if 1 left CMP BYTES, $1 - BC 12, 0, LR // ble lr + BLT CR0, LR MOVBZ 0(SRC), TMP MOVBZ TMP, 0(TGT) RET @@ -182,7 +182,7 @@ backwardtailloop: BDNZ backwardtailloop nobackwardtail: - BC 4, 5, LR // blelr cr1, return if DWORDS == 0 + BLE CR1, LR // return if DWORDS == 0 SRDCC $2,DWORDS,QWORDS // Compute number of 32B blocks and compare to 0 BNE backward32setup // If QWORDS != 0, start the 32B copy loop. @@ -190,16 +190,16 @@ backward24: // DWORDS is a value between 1-3. CMP DWORDS, $2 - MOVD -8(SRC), TMP - MOVD TMP, -8(TGT) - BC 12, 0, LR // bltlr, return if DWORDS == 1 + MOVD -8(SRC), TMP + MOVD TMP, -8(TGT) + BLT CR0, LR // return if DWORDS == 1 - MOVD -16(SRC), TMP - MOVD TMP, -16(TGT) - BC 12, 2, LR // beqlr, return if DWORDS == 2 + MOVD -16(SRC), TMP + MOVD TMP, -16(TGT) + BEQ CR0, LR // return if DWORDS == 2 - MOVD -24(SRC), TMP - MOVD TMP, -24(TGT) + MOVD -24(SRC), TMP + MOVD TMP, -24(TGT) RET backward32setup: @@ -216,5 +216,5 @@ backward32loop: STXVD2X VS32, (R0)(TGT) // store 16x2 bytes STXVD2X VS33, (IDX16)(TGT) BDNZ backward32loop - BC 12, 2, LR // beqlr, return if DWORDS == 0 + BEQ CR0, LR // return if DWORDS == 0 BR backward24 diff --git a/src/runtime/metrics/description_test.go b/src/runtime/metrics/description_test.go index 4fc65236..0ee9ea16 100644 --- a/src/runtime/metrics/description_test.go +++ b/src/runtime/metrics/description_test.go @@ -18,7 +18,7 @@ import ( "os" "regexp" "runtime/metrics" - "sort" + "slices" "strings" "testing" _ "unsafe" @@ -43,7 +43,7 @@ func TestNames(t *testing.T) { } names := runtime_readMetricNames() - sort.Strings(names) + slices.Sort(names) samples := make([]metrics.Sample, len(names)) for i, name := range names { samples[i].Name = name diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index da3d956d..563ddf4c 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -250,6 +250,11 @@ Below is the full list of supported metrics, ordered lexicographically. The number of non-default behaviors executed by the cmd/go package due to a non-default GODEBUG=gocacheverify=... setting. + /godebug/non-default-behavior/gotestjsonbuildtext:events + The number of non-default behaviors executed by the cmd/go + package due to a non-default GODEBUG=gotestjsonbuildtext=... + setting. + /godebug/non-default-behavior/gotypesalias:events The number of non-default behaviors executed by the go/types package due to a non-default GODEBUG=gotypesalias=... setting. @@ -306,6 +311,14 @@ Below is the full list of supported metrics, ordered lexicographically. The number of non-default behaviors executed by the math/rand package due to a non-default GODEBUG=randautoseed=... setting. + /godebug/non-default-behavior/randseednop:events + The number of non-default behaviors executed by the math/rand + package due to a non-default GODEBUG=randseednop=... setting. + + /godebug/non-default-behavior/rsa1024min:events + The number of non-default behaviors executed by the crypto/rsa + package due to a non-default GODEBUG=rsa1024min=... setting. + /godebug/non-default-behavior/tarinsecurepath:events The number of non-default behaviors executed by the archive/tar package due to a non-default GODEBUG=tarinsecurepath=... @@ -349,9 +362,9 @@ Below is the full list of supported metrics, ordered lexicographically. package due to a non-default GODEBUG=x509negativeserial=... setting. - /godebug/non-default-behavior/x509sha1:events + /godebug/non-default-behavior/x509rsacrt:events The number of non-default behaviors executed by the crypto/x509 - package due to a non-default GODEBUG=x509sha1=... setting. + package due to a non-default GODEBUG=x509rsacrt=... setting. /godebug/non-default-behavior/x509usefallbackroots:events The number of non-default behaviors executed by the crypto/x509 diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index ebbf0e4f..9191d86d 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -201,10 +201,10 @@ func TestReadMetrics(t *testing.T) { checkUint64(t, "/gc/heap/frees:objects", frees, mstats.Frees-tinyAllocs) // Verify that /gc/pauses:seconds is a copy of /sched/pauses/total/gc:seconds - if !reflect.DeepEqual(gcPauses.Buckets, schedPausesTotalGC.Buckets) { + if !slices.Equal(gcPauses.Buckets, schedPausesTotalGC.Buckets) { t.Errorf("/gc/pauses:seconds buckets %v do not match /sched/pauses/total/gc:seconds buckets %v", gcPauses.Buckets, schedPausesTotalGC.Counts) } - if !reflect.DeepEqual(gcPauses.Counts, schedPausesTotalGC.Counts) { + if !slices.Equal(gcPauses.Counts, schedPausesTotalGC.Counts) { t.Errorf("/gc/pauses:seconds counts %v do not match /sched/pauses/total/gc:seconds counts %v", gcPauses.Counts, schedPausesTotalGC.Counts) } } diff --git a/src/runtime/mfinal.go b/src/runtime/mfinal.go index 78313fb7..4962a63a 100644 --- a/src/runtime/mfinal.go +++ b/src/runtime/mfinal.go @@ -10,7 +10,7 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -40,11 +40,14 @@ const ( fingWake ) -var finlock mutex // protects the following variables -var fing *g // goroutine that runs finalizers -var finq *finblock // list of finalizers that are to be executed -var finc *finblock // cache of free blocks -var finptrmask [_FinBlockSize / goarch.PtrSize / 8]byte +// This runs durring the GC sweep phase. Heap memory can't be allocated while sweep is running. +var ( + finlock mutex // protects the following variables + fing *g // goroutine that runs finalizers + finq *finblock // list of finalizers that are to be executed + finc *finblock // cache of free blocks + finptrmask [_FinBlockSize / goarch.PtrSize / 8]byte +) var allfin *finblock // list of all blocks @@ -172,7 +175,7 @@ func finalizercommit(gp *g, lock unsafe.Pointer) bool { return true } -// This is the goroutine that runs all of the finalizers. +// This is the goroutine that runs all of the finalizers and cleanups. func runfinq() { var ( frame unsafe.Pointer @@ -202,6 +205,22 @@ func runfinq() { for i := fb.cnt; i > 0; i-- { f := &fb.fin[i-1] + // arg will only be nil when a cleanup has been queued. + if f.arg == nil { + var cleanup func() + fn := unsafe.Pointer(f.fn) + cleanup = *(*func())(unsafe.Pointer(&fn)) + fingStatus.Or(fingRunningFinalizer) + cleanup() + fingStatus.And(^fingRunningFinalizer) + + f.fn = nil + f.arg = nil + f.ot = nil + atomic.Store(&fb.cnt, i-1) + continue + } + var regs abi.RegArgs // The args may be passed in registers or on stack. Even for // the register case, we still need the spill slots. @@ -220,7 +239,8 @@ func runfinq() { frame = mallocgc(framesz, nil, true) framecap = framesz } - + // cleanups also have a nil fint. Cleanups should have been processed before + // reaching this point. if f.fint == nil { throw("missing type in runfinq") } @@ -330,6 +350,9 @@ func blockUntilEmptyFinalizerQueue(timeout int64) bool { // // SetFinalizer(obj, nil) clears any finalizer associated with obj. // +// New Go code should consider using [AddCleanup] instead, which is much +// less error-prone than SetFinalizer. +// // The argument obj must be a pointer to an object allocated by calling // new, by taking the address of a composite literal, or by taking the // address of a local variable. @@ -409,11 +432,6 @@ func blockUntilEmptyFinalizerQueue(timeout int64) bool { // need to use appropriate synchronization, such as mutexes or atomic updates, // to avoid read-write races. func SetFinalizer(obj any, finalizer any) { - if debug.sbrk != 0 { - // debug.sbrk never frees memory, so no finalizers run - // (and we don't have the data structures to record them). - return - } e := efaceOf(&obj) etyp := e._type if etyp == nil { @@ -426,11 +444,15 @@ func SetFinalizer(obj any, finalizer any) { if ot.Elem == nil { throw("nil elem type!") } - if inUserArenaChunk(uintptr(e.data)) { // Arena-allocated objects are not eligible for finalizers. throw("runtime.SetFinalizer: first argument was allocated into an arena") } + if debug.sbrk != 0 { + // debug.sbrk never frees memory, so no finalizers run + // (and we don't have the data structures to record them). + return + } // find the containing object base, span, _ := findObject(uintptr(e.data), 0, 0) diff --git a/src/runtime/mfinal_test.go b/src/runtime/mfinal_test.go index 87d31c47..5c93c74c 100644 --- a/src/runtime/mfinal_test.go +++ b/src/runtime/mfinal_test.go @@ -5,6 +5,7 @@ package runtime_test import ( + "internal/asan" "runtime" "testing" "time" @@ -165,6 +166,9 @@ func adjChunks() (*objtype, *objtype) { // Make sure an empty slice on the stack doesn't pin the next object in memory. func TestEmptySlice(t *testing.T) { + if asan.Enabled { + t.Skip("skipping with -asan: test assumes exact size class alignment, but asan redzone breaks that assumption") + } x, y := adjChunks() // the pointer inside xs points to y. @@ -194,6 +198,9 @@ func adjStringChunk() (string, *objtype) { // Make sure an empty string on the stack doesn't pin the next object in memory. func TestEmptyString(t *testing.T) { + if asan.Enabled { + t.Skip("skipping with -asan: test assumes exact size class alignment, but asan redzone breaks that assumption") + } x, y := adjStringChunk() ss := x[objsize:] // change objsize to objsize-1 and the test passes diff --git a/src/runtime/mfixalloc.go b/src/runtime/mfixalloc.go index 7760ada3..be977af7 100644 --- a/src/runtime/mfixalloc.go +++ b/src/runtime/mfixalloc.go @@ -9,7 +9,7 @@ package runtime import ( - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -27,7 +27,7 @@ import ( // smashed by freeing and reallocating. // // Consider marking fixalloc'd types not in heap by embedding -// runtime/internal/sys.NotInHeap. +// internal/runtime/sys.NotInHeap. type fixalloc struct { size uintptr first func(arg, p unsafe.Pointer) // called first time p is returned diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index f72edc2a..48001cfd 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -221,7 +221,6 @@ var gcphase uint32 // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // // Do not remove or change the type signature. // See go.dev/issue/67401. @@ -336,11 +335,12 @@ type workType struct { // bytesMarked is the number of bytes marked this cycle. This // includes bytes blackened in scanned objects, noscan objects - // that go straight to black, and permagrey objects scanned by - // markroot during the concurrent scan phase. This is updated - // atomically during the cycle. Updates may be batched - // arbitrarily, since the value is only read at the end of the - // cycle. + // that go straight to black, objects allocated as black during + // the cycle, and permagrey objects scanned by markroot during + // the concurrent scan phase. + // + // This is updated atomically during the cycle. Updates may be batched + // arbitrarily, since the value is only read at the end of the cycle. // // Because of benign races during marking, this number may not // be the exact number of marked bytes, but it should be very @@ -639,6 +639,17 @@ func gcStart(trigger gcTrigger) { releasem(mp) mp = nil + if gp := getg(); gp.syncGroup != nil { + // Disassociate the G from its synctest bubble while allocating. + // This is less elegant than incrementing the group's active count, + // but avoids any contamination between GC and synctest. + sg := gp.syncGroup + gp.syncGroup = nil + defer func() { + gp.syncGroup = sg + }() + } + // Pick up the remaining unswept/not being swept spans concurrently // // This shouldn't happen if we're being invoked in background @@ -1774,8 +1785,12 @@ func boring_registerCache(p unsafe.Pointer) { //go:linkname unique_runtime_registerUniqueMapCleanup unique.runtime_registerUniqueMapCleanup func unique_runtime_registerUniqueMapCleanup(f func()) { + // Create the channel on the system stack so it doesn't inherit the current G's + // synctest bubble (if any). + systemstack(func() { + uniqueMapCleanup = make(chan struct{}, 1) + }) // Start the goroutine in the runtime so it's counted as a system goroutine. - uniqueMapCleanup = make(chan struct{}, 1) go func(cleanup func()) { for { <-uniqueMapCleanup @@ -1908,7 +1923,7 @@ func gcTestIsReachable(ptrs ...unsafe.Pointer) (mask uint64) { s := (*specialReachable)(mheap_.specialReachableAlloc.alloc()) unlock(&mheap_.speciallock) s.special.kind = _KindSpecialReachable - if !addspecial(p, &s.special) { + if !addspecial(p, &s.special, false) { throw("already have a reachable special (duplicate pointer?)") } specials[i] = s diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 61e917df..823b2bd7 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -10,7 +10,7 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -178,6 +178,8 @@ func markroot(gcw *gcWork, i uint32, flushBgCredit bool) int64 { case i == fixedRootFinalizers: for fb := allfin; fb != nil; fb = fb.alllink { cnt := uintptr(atomic.Load(&fb.cnt)) + // Finalizers that contain cleanups only have fn set. None of the other + // fields are necessary. scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw, nil) } @@ -401,6 +403,10 @@ func markrootSpans(gcw *gcWork, shard int) { // The special itself is a root. spw := (*specialWeakHandle)(unsafe.Pointer(sp)) scanblock(uintptr(unsafe.Pointer(&spw.handle)), goarch.PtrSize, &oneptrmask[0], gcw, nil) + case _KindSpecialCleanup: + spc := (*specialCleanup)(unsafe.Pointer(sp)) + // The special itself is a root. + scanblock(uintptr(unsafe.Pointer(&spc.fn)), goarch.PtrSize, &oneptrmask[0], gcw, nil) } } unlock(&s.speciallock) @@ -422,6 +428,17 @@ func gcAssistAlloc(gp *g) { return } + if gp := getg(); gp.syncGroup != nil { + // Disassociate the G from its synctest bubble while allocating. + // This is less elegant than incrementing the group's active count, + // but avoids any contamination between GC assist and synctest. + sg := gp.syncGroup + gp.syncGroup = nil + defer func() { + gp.syncGroup = sg + }() + } + // This extremely verbose boolean indicates whether we've // entered mark assist from the perspective of the tracer. // @@ -945,31 +962,12 @@ func scanstack(gp *g, gcw *gcWork) int64 { println() printunlock() } - gcdata := r.gcdata() - var s *mspan - if r.useGCProg() { - // This path is pretty unlikely, an object large enough - // to have a GC program allocated on the stack. - // We need some space to unpack the program into a straight - // bitmask, which we allocate/free here. - // TODO: it would be nice if there were a way to run a GC - // program without having to store all its bits. We'd have - // to change from a Lempel-Ziv style program to something else. - // Or we can forbid putting objects on stacks if they require - // a gc program (see issue 27447). - s = materializeGCProg(r.ptrdata(), gcdata) - gcdata = (*byte)(unsafe.Pointer(s.startAddr)) - } - + ptrBytes, gcData := r.gcdata() b := state.stack.lo + uintptr(obj.off) if conservative { - scanConservative(b, r.ptrdata(), gcdata, gcw, &state) + scanConservative(b, ptrBytes, gcData, gcw, &state) } else { - scanblock(b, r.ptrdata(), gcdata, gcw, &state) - } - - if s != nil { - dematerializeGCProg(s) + scanblock(b, ptrBytes, gcData, gcw, &state) } } @@ -1694,6 +1692,10 @@ func gcmarknewobject(span *mspan, obj uintptr) { if useCheckmark { // The world should be stopped so this should not happen. throw("gcmarknewobject called while doing checkmark") } + if gcphase == _GCmarktermination { + // Check this here instead of on the hot path. + throw("mallocgc called with gcphase == _GCmarktermination") + } // Mark object. objIndex := span.objIndex(obj) diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index cda87fe9..3e80fae4 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -752,6 +752,17 @@ func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { return nil, now } + if c.dedicatedMarkWorkersNeeded.Load() <= 0 && c.fractionalUtilizationGoal == 0 { + // No current need for dedicated workers, and no need at all for + // fractional workers. Check before trying to acquire a worker; when + // GOMAXPROCS is large, that can be expensive and is often unnecessary. + // + // When a dedicated worker stops running, the gcBgMarkWorker loop notes + // the need for the worker before returning it to the pool. If we don't + // see the need now, we wouldn't have found it in the pool anyway. + return nil, now + } + // Grab a worker before we commit to running below. node := (*gcBgMarkWorkerNode)(gcBgMarkWorkerPool.pop()) if node == nil { diff --git a/src/runtime/mgcscavenge.go b/src/runtime/mgcscavenge.go index 4f0bd9c2..3d869ecd 100644 --- a/src/runtime/mgcscavenge.go +++ b/src/runtime/mgcscavenge.go @@ -93,7 +93,7 @@ package runtime import ( "internal/goos" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/mgcstack.go b/src/runtime/mgcstack.go index f4a83f5f..bc5911f1 100644 --- a/src/runtime/mgcstack.go +++ b/src/runtime/mgcstack.go @@ -96,7 +96,7 @@ package runtime import ( "internal/goarch" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index f53330a5..b6890bac 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -25,7 +25,6 @@ package runtime import ( - "internal/abi" "internal/runtime/atomic" "unsafe" ) @@ -565,7 +564,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool { } if hasFinAndRevived { // Pass 2: queue all finalizers and clear any weak handles. Weak handles are cleared - // before finalization as specified by the internal/weak package. See the documentation + // before finalization as specified by the weak package. See the documentation // for that package for more details. for siter.valid() && uintptr(siter.s.offset) < endOffset { // Find the exact byte for which the special was setup @@ -818,18 +817,6 @@ func (sl *sweepLocked) sweep(preserve bool) bool { } else { mheap_.freeSpan(s) } - if s.largeType != nil && s.largeType.TFlag&abi.TFlagUnrolledBitmap != 0 { - // The unrolled GCProg bitmap is allocated separately. - // Free the space for the unrolled bitmap. - systemstack(func() { - s := spanOf(uintptr(unsafe.Pointer(s.largeType))) - mheap_.freeManual(s, spanAllocPtrScalarBits) - }) - // Make sure to zero this pointer without putting the old - // value in a write buffer, as the old value might be an - // invalid pointer. See arena.go:(*mheap).allocUserArenaChunk. - *(*uintptr)(unsafe.Pointer(&s.largeType)) = 0 - } return true } @@ -855,7 +842,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool { // pointer to that object and marked it. func (s *mspan) reportZombies() { printlock() - print("runtime: marked free object in span ", s, ", elemsize=", s.elemsize, " freeindex=", s.freeindex, " (bad use of unsafe.Pointer? try -d=checkptr)\n") + print("runtime: marked free object in span ", s, ", elemsize=", s.elemsize, " freeindex=", s.freeindex, " (bad use of unsafe.Pointer or having race conditions? try -d=checkptr or -race)\n") mbits := s.markBitsForBase() abits := s.allocBitsForIndex(0) for i := uintptr(0); i < uintptr(s.nelems); i++ { diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go index b91a6bd4..2d66fa40 100644 --- a/src/runtime/mgcwork.go +++ b/src/runtime/mgcwork.go @@ -7,7 +7,7 @@ package runtime import ( "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index bfca2d10..e058dd84 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -12,7 +12,7 @@ import ( "internal/cpu" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -204,6 +204,7 @@ type mheap struct { spanalloc fixalloc // allocator for span* cachealloc fixalloc // allocator for mcache* specialfinalizeralloc fixalloc // allocator for specialfinalizer* + specialCleanupAlloc fixalloc // allocator for specialcleanup* specialprofilealloc fixalloc // allocator for specialprofile* specialReachableAlloc fixalloc // allocator for specialReachable specialPinCounterAlloc fixalloc // allocator for specialPinCounter @@ -230,6 +231,12 @@ type mheap struct { readyList mSpanList } + // cleanupID is a counter which is incremented each time a cleanup special is added + // to a span. It's used to create globally unique identifiers for individual cleanup. + // cleanupID is protected by mheap_.lock. It should only be incremented while holding + // the lock. + cleanupID uint64 + unused *specialfinalizer // never set, just here to force the specialfinalizer type into DWARF } @@ -743,6 +750,7 @@ func (h *mheap) init() { h.spanalloc.init(unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys) h.cachealloc.init(unsafe.Sizeof(mcache{}), nil, nil, &memstats.mcache_sys) h.specialfinalizeralloc.init(unsafe.Sizeof(specialfinalizer{}), nil, nil, &memstats.other_sys) + h.specialCleanupAlloc.init(unsafe.Sizeof(specialCleanup{}), nil, nil, &memstats.other_sys) h.specialprofilealloc.init(unsafe.Sizeof(specialprofile{}), nil, nil, &memstats.other_sys) h.specialReachableAlloc.init(unsafe.Sizeof(specialReachable{}), nil, nil, &memstats.other_sys) h.specialPinCounterAlloc.init(unsafe.Sizeof(specialPinCounter{}), nil, nil, &memstats.other_sys) @@ -1368,7 +1376,7 @@ HaveSpan: // Trace the span alloc. if traceAllocFreeEnabled() { - trace := traceTryAcquire() + trace := traceAcquire() if trace.ok() { trace.SpanAlloc(s) traceRelease(trace) @@ -1556,7 +1564,7 @@ func (h *mheap) freeSpan(s *mspan) { systemstack(func() { // Trace the span free. if traceAllocFreeEnabled() { - trace := traceTryAcquire() + trace := traceAcquire() if trace.ok() { trace.SpanFree(s) traceRelease(trace) @@ -1595,7 +1603,7 @@ func (h *mheap) freeSpan(s *mspan) { func (h *mheap) freeManual(s *mspan, typ spanAllocType) { // Trace the span free. if traceAllocFreeEnabled() { - trace := traceTryAcquire() + trace := traceAcquire() if trace.ok() { trace.SpanFree(s) traceRelease(trace) @@ -1824,12 +1832,14 @@ const ( // _KindSpecialPinCounter is a special used for objects that are pinned // multiple times _KindSpecialPinCounter = 5 + // _KindSpecialCleanup is for tracking cleanups. + _KindSpecialCleanup = 6 ) type special struct { _ sys.NotInHeap next *special // linked list in span - offset uint16 // span offset of object + offset uintptr // span offset of object kind byte // kind of special } @@ -1849,13 +1859,13 @@ func spanHasNoSpecials(s *mspan) { atomic.And8(&ha.pageSpecials[arenaPage/8], ^(uint8(1) << (arenaPage % 8))) } -// Adds the special record s to the list of special records for +// addspecial adds the special record s to the list of special records for // the object p. All fields of s should be filled in except for // offset & next, which this routine will fill in. // Returns true if the special was successfully added, false otherwise. // (The add will fail only if a record with the same p and s->kind -// already exists.) -func addspecial(p unsafe.Pointer, s *special) bool { +// already exists unless force is set to true.) +func addspecial(p unsafe.Pointer, s *special, force bool) bool { span := spanOfHeap(uintptr(p)) if span == nil { throw("addspecial on invalid pointer") @@ -1874,9 +1884,9 @@ func addspecial(p unsafe.Pointer, s *special) bool { // Find splice point, check for existing record. iter, exists := span.specialFindSplicePoint(offset, kind) - if !exists { + if !exists || force { // Splice in record, fill in offset. - s.offset = uint16(offset) + s.offset = offset s.next = *iter *iter = s spanHasSpecials(span) @@ -1884,7 +1894,10 @@ func addspecial(p unsafe.Pointer, s *special) bool { unlock(&span.speciallock) releasem(mp) - return !exists // already exists + // We're converting p to a uintptr and looking it up, and we + // don't want it to die and get swept while we're doing so. + KeepAlive(p) + return !exists || force // already exists or addition was forced } // Removes the Special record of the given kind for the object p. @@ -1968,7 +1981,7 @@ func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *p s.nret = nret s.fint = fint s.ot = ot - if addspecial(p, &s.special) { + if addspecial(p, &s.special, false) { // This is responsible for maintaining the same // GC-related invariants as markrootSpans in any // situation where it's possible that markrootSpans @@ -2008,6 +2021,50 @@ func removefinalizer(p unsafe.Pointer) { unlock(&mheap_.speciallock) } +// The described object has a cleanup set for it. +type specialCleanup struct { + _ sys.NotInHeap + special special + fn *funcval + // Globally unique ID for the cleanup, obtained from mheap_.cleanupID. + id uint64 +} + +// addCleanup attaches a cleanup function to the object. Multiple +// cleanups are allowed on an object, and even the same pointer. +// A cleanup id is returned which can be used to uniquely identify +// the cleanup. +func addCleanup(p unsafe.Pointer, f *funcval) uint64 { + lock(&mheap_.speciallock) + s := (*specialCleanup)(mheap_.specialCleanupAlloc.alloc()) + mheap_.cleanupID++ + id := mheap_.cleanupID + unlock(&mheap_.speciallock) + s.special.kind = _KindSpecialCleanup + s.fn = f + s.id = id + + mp := acquirem() + addspecial(p, &s.special, true) + // This is responsible for maintaining the same + // GC-related invariants as markrootSpans in any + // situation where it's possible that markrootSpans + // has already run but mark termination hasn't yet. + if gcphase != _GCoff { + gcw := &mp.p.ptr().gcw + // Mark the cleanup itself, since the + // special isn't part of the GC'd heap. + scanblock(uintptr(unsafe.Pointer(&s.fn)), goarch.PtrSize, &oneptrmask[0], gcw, nil) + } + releasem(mp) + // Keep f alive. There's a window in this function where it's + // only reachable via the special while the special hasn't been + // added to the specials list yet. This is similar to a bug + // discovered for weak handles, see #70455. + KeepAlive(f) + return id +} + // The described object has a weak pointer. // // Weak pointers in the GC have the following invariants: @@ -2040,12 +2097,12 @@ type specialWeakHandle struct { handle *atomic.Uintptr } -//go:linkname internal_weak_runtime_registerWeakPointer internal/weak.runtime_registerWeakPointer +//go:linkname internal_weak_runtime_registerWeakPointer weak.runtime_registerWeakPointer func internal_weak_runtime_registerWeakPointer(p unsafe.Pointer) unsafe.Pointer { return unsafe.Pointer(getOrAddWeakHandle(unsafe.Pointer(p))) } -//go:linkname internal_weak_runtime_makeStrongFromWeak internal/weak.runtime_makeStrongFromWeak +//go:linkname internal_weak_runtime_makeStrongFromWeak weak.runtime_makeStrongFromWeak func internal_weak_runtime_makeStrongFromWeak(u unsafe.Pointer) unsafe.Pointer { handle := (*atomic.Uintptr)(u) @@ -2156,7 +2213,7 @@ func getOrAddWeakHandle(p unsafe.Pointer) *atomic.Uintptr { s.special.kind = _KindSpecialWeakHandle s.handle = handle handle.Store(uintptr(p)) - if addspecial(p, &s.special) { + if addspecial(p, &s.special, false) { // This is responsible for maintaining the same // GC-related invariants as markrootSpans in any // situation where it's possible that markrootSpans @@ -2251,7 +2308,7 @@ func setprofilebucket(p unsafe.Pointer, b *bucket) { unlock(&mheap_.speciallock) s.special.kind = _KindSpecialProfile s.b = b - if !addspecial(p, &s.special) { + if !addspecial(p, &s.special, false) { throw("setprofilebucket: profile already set") } } @@ -2328,6 +2385,15 @@ func freeSpecial(s *special, p unsafe.Pointer, size uintptr) { lock(&mheap_.speciallock) mheap_.specialPinCounterAlloc.free(unsafe.Pointer(s)) unlock(&mheap_.speciallock) + case _KindSpecialCleanup: + sc := (*specialCleanup)(unsafe.Pointer(s)) + // Cleanups, unlike finalizers, do not resurrect the objects + // they're attached to, so we only need to pass the cleanup + // function, not the object. + queuefinalizer(nil, sc.fn, 0, nil, nil) + lock(&mheap_.speciallock) + mheap_.specialCleanupAlloc.free(unsafe.Pointer(sc)) + unlock(&mheap_.speciallock) default: throw("bad special kind") panic("not reached") diff --git a/src/runtime/mklockrank.go b/src/runtime/mklockrank.go index 3391afc6..e4a749dd 100644 --- a/src/runtime/mklockrank.go +++ b/src/runtime/mklockrank.go @@ -95,6 +95,9 @@ NONE < itab < reflectOffs; +# Synctest +hchan, root, timers, timer, notifyList, reflectOffs < synctest; + # User arena state NONE < userArenaState; @@ -145,6 +148,7 @@ gcBitsArenas, profInsert, profMemFuture, spanSetSpine, + synctest, fin, root # Anything that can grow the stack can acquire STACKGROW. diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index 17544d6b..08500a90 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -264,19 +264,6 @@ func genAMD64() { l.save() - // Apparently, the signal handling code path in darwin kernel leaves - // the upper bits of Y registers in a dirty state, which causes - // many SSE operations (128-bit and narrower) become much slower. - // Clear the upper bits to get to a clean state. See issue #37174. - // It is safe here as Go code don't use the upper bits of Y registers. - p("#ifdef GOOS_darwin") - p("#ifndef hasAVX") - p("CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0") - p("JE 2(PC)") - p("#endif") - p("VZEROUPPER") - p("#endif") - lSSE.save() p("CALL ·asyncPreempt2(SB)") lSSE.restore() diff --git a/src/runtime/mpagecache.go b/src/runtime/mpagecache.go index 245b0cbf..0ed3e80c 100644 --- a/src/runtime/mpagecache.go +++ b/src/runtime/mpagecache.go @@ -5,7 +5,7 @@ package runtime import ( - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/mpallocbits.go b/src/runtime/mpallocbits.go index d8a9d257..e8e70f36 100644 --- a/src/runtime/mpallocbits.go +++ b/src/runtime/mpallocbits.go @@ -5,7 +5,7 @@ package runtime import ( - "runtime/internal/sys" + "internal/runtime/sys" ) // pageBits is a bitmap representing one bit per page in a palloc chunk. diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index ee3e59a9..3cf8dc81 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -12,7 +12,7 @@ import ( "internal/goarch" "internal/profilerecord" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -49,7 +49,7 @@ const ( // desired maximum number of frames after expansion. // This should be at least as large as the largest skip value // used for profiling; otherwise stacks may be truncated inconsistently - maxSkip = 5 + maxSkip = 6 // maxProfStackDepth is the highest valid value for debug.profstackdepth. // It's used for the bucket.stk func. @@ -242,7 +242,7 @@ func newBucket(typ bucketType, nstk int) *bucket { return b } -// stk returns the slice in b holding the stack. The caller can asssume that the +// stk returns the slice in b holding the stack. The caller can assume that the // backing array is immutable. func (b *bucket) stk() []uintptr { stk := (*[maxProfStackDepth]uintptr)(add(unsafe.Pointer(b), unsafe.Sizeof(*b))) @@ -444,7 +444,7 @@ func mProf_Malloc(mp *m, p unsafe.Pointer, size uintptr) { } // Only use the part of mp.profStack we need and ignore the extra space // reserved for delayed inline expansion with frame pointer unwinding. - nstk := callers(4, mp.profStack[:debug.profstackdepth]) + nstk := callers(5, mp.profStack[:debug.profstackdepth]) index := (mProfCycle.read() + 2) % uint32(len(memRecord{}.future)) b := stkbucket(memProfile, size, mp.profStack[:nstk], true) @@ -718,12 +718,13 @@ type mLockProfile struct { pending uintptr // *mutex that experienced contention (to be traceback-ed) cycles int64 // cycles attributable to "pending" (if set), otherwise to "stack" cyclesLost int64 // contention for which we weren't able to record a call stack + haveStack bool // stack and cycles are to be added to the mutex profile disabled bool // attribute all time to "lost" } func (prof *mLockProfile) recordLock(cycles int64, l *mutex) { - if cycles <= 0 { - return + if cycles < 0 { + cycles = 0 } if prof.disabled { @@ -745,6 +746,9 @@ func (prof *mLockProfile) recordLock(cycles int64, l *mutex) { // We can only store one call stack for runtime-internal lock contention // on this M, and we've already got one. Decide which should stay, and // add the other to the report for runtime._LostContendedRuntimeLock. + if cycles == 0 { + return + } prevScore := uint64(cheaprand64()) % uint64(prev) thisScore := uint64(cheaprand64()) % uint64(cycles) if prevScore > thisScore { @@ -769,7 +773,7 @@ func (prof *mLockProfile) recordUnlock(l *mutex) { if uintptr(unsafe.Pointer(l)) == prof.pending { prof.captureStack() } - if gp := getg(); gp.m.locks == 1 && gp.m.mLockProfile.cycles != 0 { + if gp := getg(); gp.m.locks == 1 && gp.m.mLockProfile.haveStack { prof.store() } } @@ -795,6 +799,7 @@ func (prof *mLockProfile) captureStack() { skip += 1 // runtime.unlockWithRank.func1 } prof.pending = 0 + prof.haveStack = true prof.stack[0] = logicalStackSentinel if debug.runtimeContentionStacks.Load() == 0 { @@ -805,8 +810,8 @@ func (prof *mLockProfile) captureStack() { var nstk int gp := getg() - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { var u unwinder u.initAt(pc, sp, 0, gp, unwindSilentErrors|unwindJumpStack) @@ -835,6 +840,7 @@ func (prof *mLockProfile) store() { cycles, lost := prof.cycles, prof.cyclesLost prof.cycles, prof.cyclesLost = 0, 0 + prof.haveStack = false rate := int64(atomic.Load64(&mutexprofilerate)) saveBlockEventStack(cycles, rate, prof.stack[:nstk], mutexProfile) @@ -1074,7 +1080,7 @@ func copyMemProfileRecord(dst *MemProfileRecord, src profilerecord.MemProfileRec dst.AllocObjects = src.AllocObjects dst.FreeObjects = src.FreeObjects if raceenabled { - racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), getcallerpc(), abi.FuncPCABIInternal(MemProfile)) + racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), sys.GetCallerPC(), abi.FuncPCABIInternal(MemProfile)) } if msanenabled { msanwrite(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0)) @@ -1188,7 +1194,7 @@ func copyBlockProfileRecord(dst *BlockProfileRecord, src profilerecord.BlockProf dst.Count = src.Count dst.Cycles = src.Cycles if raceenabled { - racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), getcallerpc(), abi.FuncPCABIInternal(BlockProfile)) + racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), sys.GetCallerPC(), abi.FuncPCABIInternal(BlockProfile)) } if msanenabled { msanwrite(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0)) @@ -1397,8 +1403,8 @@ func goroutineProfileWithLabelsConcurrent(p []profilerecord.StackRecord, labels } // Save current goroutine. - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { saveg(pc, sp, ourg, &p[0], pcbuf) }) @@ -1593,8 +1599,8 @@ func goroutineProfileWithLabelsSync(p []profilerecord.StackRecord, labels []unsa r, lbl := p, labels // Save current goroutine. - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { saveg(pc, sp, gp, &r[0], pcbuf) }) @@ -1696,8 +1702,8 @@ func Stack(buf []byte, all bool) int { n := 0 if len(buf) > 0 { gp := getg() - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { g0 := getg() // Force traceback=1 to override GOTRACEBACK setting, diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index 7b37d91b..8d65a81e 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -8,7 +8,7 @@ package runtime import ( "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -31,7 +31,7 @@ import ( // poll without blocking. If delta > 0, block for up to delta nanoseconds. // Return a list of goroutines built by calling netpollready, // and a delta to add to netpollWaiters when all goroutines are ready. -// This will never return an empty list with a non-zero delta. +// This must never return an empty list with a non-zero delta. // // func netpollBreak() // Wake up the network poller, assumed to be blocked in netpoll. @@ -714,7 +714,7 @@ func (c *pollCache) alloc() *pollDesc { // makeArg converts pd to an interface{}. // makeArg does not do any allocation. Normally, such // a conversion requires an allocation because pointers to -// types which embed runtime/internal/sys.NotInHeap (which pollDesc is) +// types which embed internal/runtime/sys.NotInHeap (which pollDesc is) // must be stored in interfaces indirectly. See issue 42076. func (pd *pollDesc) makeArg() (i any) { x := (*eface)(unsafe.Pointer(&i)) diff --git a/src/runtime/netpoll_aix.go b/src/runtime/netpoll_aix.go index 2df5a571..4cf34d1c 100644 --- a/src/runtime/netpoll_aix.go +++ b/src/runtime/netpoll_aix.go @@ -148,7 +148,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds diff --git a/src/runtime/netpoll_epoll.go b/src/runtime/netpoll_epoll.go index ff6e0b5f..c43bab08 100644 --- a/src/runtime/netpoll_epoll.go +++ b/src/runtime/netpoll_epoll.go @@ -89,7 +89,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds diff --git a/src/runtime/netpoll_kqueue.go b/src/runtime/netpoll_kqueue.go index 6cd80d5c..db4dddc2 100644 --- a/src/runtime/netpoll_kqueue.go +++ b/src/runtime/netpoll_kqueue.go @@ -80,7 +80,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds @@ -126,10 +129,11 @@ retry: ev := &events[i] if isWakeup(ev) { - if delay != 0 { + isBlocking := delay != 0 + processWakeupEvent(kq, isBlocking) + if isBlocking { // netpollBreak could be picked up by a nonblocking poll. - // Only call drainWakeupEvent and reset the netpollWakeSig if blocking. - drainWakeupEvent(kq) + // Only reset the netpollWakeSig if blocking. netpollWakeSig.Store(0) } continue diff --git a/src/runtime/netpoll_kqueue_event.go b/src/runtime/netpoll_kqueue_event.go index d5f783e6..852a00a5 100644 --- a/src/runtime/netpoll_kqueue_event.go +++ b/src/runtime/netpoll_kqueue_event.go @@ -16,7 +16,7 @@ func addWakeupEvent(kq int32) { ev := keventt{ ident: kqIdent, filter: _EVFILT_USER, - flags: _EV_ADD, + flags: _EV_ADD | _EV_CLEAR, } for { n := kevent(kq, &ev, 1, nil, 0, nil) @@ -38,7 +38,6 @@ func wakeNetpoll(kq int32) { ev := keventt{ ident: kqIdent, filter: _EVFILT_USER, - flags: _EV_ENABLE, fflags: _NOTE_TRIGGER, } for { @@ -66,13 +65,11 @@ func isWakeup(ev *keventt) bool { return false } -func drainWakeupEvent(kq int32) { - ev := keventt{ - ident: kqIdent, - filter: _EVFILT_USER, - flags: _EV_DISABLE, +func processWakeupEvent(kq int32, isBlocking bool) { + if !isBlocking { + // Got a wrong thread, relay + wakeNetpoll(kq) } - kevent(kq, &ev, 1, nil, 0, nil) } func netpollIsPollDescriptor(fd uintptr) bool { diff --git a/src/runtime/netpoll_kqueue_pipe.go b/src/runtime/netpoll_kqueue_pipe.go index 98f73e84..cf1e2afa 100644 --- a/src/runtime/netpoll_kqueue_pipe.go +++ b/src/runtime/netpoll_kqueue_pipe.go @@ -63,7 +63,11 @@ func isWakeup(ev *keventt) bool { return false } -func drainWakeupEvent(_ int32) { +func processWakeupEvent(_ int32, isBlocking bool) { + // Only drain if blocking. + if !isBlocking { + return + } var buf [16]byte read(int32(netpollBreakRd), noescape(unsafe.Pointer(&buf[0])), int32(len(buf))) } diff --git a/src/runtime/netpoll_solaris.go b/src/runtime/netpoll_solaris.go index fddc2900..90459c08 100644 --- a/src/runtime/netpoll_solaris.go +++ b/src/runtime/netpoll_solaris.go @@ -215,7 +215,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds diff --git a/src/runtime/netpoll_stub.go b/src/runtime/netpoll_stub.go index c1bda3fa..dc5d708b 100644 --- a/src/runtime/netpoll_stub.go +++ b/src/runtime/netpoll_stub.go @@ -32,7 +32,9 @@ func netpollBreak() { } // Polls for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. func netpoll(delay int64) (gList, int32) { // Implementation for platforms that do not support // integrated network poller. diff --git a/src/runtime/netpoll_wasip1.go b/src/runtime/netpoll_wasip1.go index e6b299a2..c7e66a64 100644 --- a/src/runtime/netpoll_wasip1.go +++ b/src/runtime/netpoll_wasip1.go @@ -209,7 +209,7 @@ func netpoll(delay int64) (gList, int32) { retry: var nevents size - errno := poll_oneoff(unsafe.Pointer(&pollsubs[0]), unsafe.Pointer(&evts[0]), uint32(len(pollsubs)), unsafe.Pointer(&nevents)) + errno := poll_oneoff(&pollsubs[0], &evts[0], uint32(len(pollsubs)), &nevents) if errno != 0 { if errno != _EINTR { println("errno=", errno, " len(pollsubs)=", len(pollsubs)) diff --git a/src/runtime/netpoll_windows.go b/src/runtime/netpoll_windows.go index c3c10af7..fb35d41c 100644 --- a/src/runtime/netpoll_windows.go +++ b/src/runtime/netpoll_windows.go @@ -144,7 +144,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds diff --git a/src/runtime/norace_linux_test.go b/src/runtime/nosan_linux_test.go similarity index 76% rename from src/runtime/norace_linux_test.go rename to src/runtime/nosan_linux_test.go index 3521b246..5c99591a 100644 --- a/src/runtime/norace_linux_test.go +++ b/src/runtime/nosan_linux_test.go @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// The file contains tests that cannot run under race detector for some reason. +// The file contains tests that cannot run under race detector (or asan or msan) for some reason. // -//go:build !race +//go:build !race && !asan && !msan package runtime_test @@ -23,7 +23,7 @@ func newOSProcCreated() { newOSProcDone = true } -// Can't be run with -race because it inserts calls into newOSProcCreated() +// Can't be run with -race, -asan, or -msan because it inserts calls into newOSProcCreated() // that require a valid G/M. func TestNewOSProc0(t *testing.T) { runtime.NewOSProc0(0x800000, unsafe.Pointer(abi.FuncPCABIInternal(newOSProcCreated))) diff --git a/src/runtime/note_js.go b/src/runtime/note_js.go new file mode 100644 index 00000000..be43fa42 --- /dev/null +++ b/src/runtime/note_js.go @@ -0,0 +1,40 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// sleep and wakeup on one-time events. +// before any calls to notesleep or notewakeup, +// must call noteclear to initialize the Note. +// then, exactly one thread can call notesleep +// and exactly one thread can call notewakeup (once). +// once notewakeup has been called, the notesleep +// will return. future notesleep will return immediately. +// subsequent noteclear must be called only after +// previous notesleep has returned, e.g. it's disallowed +// to call noteclear straight after notewakeup. +// +// notetsleep is like notesleep but wakes up after +// a given number of nanoseconds even if the event +// has not yet happened. if a goroutine uses notetsleep to +// wake up early, it must wait to call noteclear until it +// can be sure that no other goroutine is calling +// notewakeup. +// +// notesleep/notetsleep are generally called on g0, +// notetsleepg is similar to notetsleep but is called on user g. +type note struct { + status int32 + + // The G waiting on this note. + gp *g + + // Deadline, if any. 0 indicates no timeout. + deadline int64 + + // allprev and allnext are used to form the allDeadlineNotes linked + // list. These are unused if there is no deadline. + allprev *note + allnext *note +} diff --git a/src/runtime/note_other.go b/src/runtime/note_other.go new file mode 100644 index 00000000..7f62c1c6 --- /dev/null +++ b/src/runtime/note_other.go @@ -0,0 +1,33 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !js + +package runtime + +// sleep and wakeup on one-time events. +// before any calls to notesleep or notewakeup, +// must call noteclear to initialize the Note. +// then, exactly one thread can call notesleep +// and exactly one thread can call notewakeup (once). +// once notewakeup has been called, the notesleep +// will return. future notesleep will return immediately. +// subsequent noteclear must be called only after +// previous notesleep has returned, e.g. it's disallowed +// to call noteclear straight after notewakeup. +// +// notetsleep is like notesleep but wakes up after +// a given number of nanoseconds even if the event +// has not yet happened. if a goroutine uses notetsleep to +// wake up early, it must wait to call noteclear until it +// can be sure that no other goroutine is calling +// notewakeup. +// +// notesleep/notetsleep are generally called on g0, +// notetsleepg is similar to notetsleep but is called on user g. +type note struct { + // Futex-based impl treats it as uint32 key, + // while sema-based impl as M* waitm. + key uintptr +} diff --git a/src/runtime/os2_aix.go b/src/runtime/os2_aix.go index 0d200792..51758bd3 100644 --- a/src/runtime/os2_aix.go +++ b/src/runtime/os2_aix.go @@ -11,6 +11,7 @@ package runtime import ( + "internal/runtime/sys" "unsafe" ) @@ -182,10 +183,10 @@ func syscall0(fn *libFunc) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -213,10 +214,10 @@ func syscall1(fn *libFunc, a0 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -245,10 +246,10 @@ func syscall2(fn *libFunc, a0, a1 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -277,10 +278,10 @@ func syscall3(fn *libFunc, a0, a1, a2 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -309,10 +310,10 @@ func syscall4(fn *libFunc, a0, a1, a2, a3 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -341,10 +342,10 @@ func syscall5(fn *libFunc, a0, a1, a2, a3, a4 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -373,10 +374,10 @@ func syscall6(fn *libFunc, a0, a1, a2, a3, a4, a5 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 430d1865..0ecbea7a 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -6,6 +6,7 @@ package runtime import ( "internal/abi" + "internal/stringslite" "unsafe" ) @@ -191,14 +192,10 @@ func getPageSize() uintptr { return 0 } -var urandom_dev = []byte("/dev/urandom\x00") - //go:nosplit func readRandom(r []byte) int { - fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) - n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) - closefd(fd) - return int(n) + arc4random_buf(unsafe.Pointer(&r[0]), int32(len(r))) + return len(r) } func goenvs() { @@ -465,10 +462,7 @@ func sysargs(argc int32, argv **byte) { executablePath = gostringnocopy(argv_index(argv, n+1)) // strip "executable_path=" prefix if available, it's added after OS X 10.11. - const prefix = "executable_path=" - if len(executablePath) > len(prefix) && executablePath[:len(prefix)] == prefix { - executablePath = executablePath[len(prefix):] - } + executablePath = stringslite.TrimPrefix(executablePath, "executable_path=") } func signalM(mp *m, sig int) { diff --git a/src/runtime/os_dragonfly.go b/src/runtime/os_dragonfly.go index 2aeea177..a02696eb 100644 --- a/src/runtime/os_dragonfly.go +++ b/src/runtime/os_dragonfly.go @@ -19,7 +19,9 @@ const ( _SIG_SETMASK = 3 ) -type mOS struct{} +type mOS struct { + waitsema uint32 // semaphore for parking on locks +} //go:noescape func lwp_create(param *lwpparams) int32 diff --git a/src/runtime/os_freebsd.go b/src/runtime/os_freebsd.go index d0d6f14f..3187317a 100644 --- a/src/runtime/os_freebsd.go +++ b/src/runtime/os_freebsd.go @@ -10,7 +10,9 @@ import ( "unsafe" ) -type mOS struct{} +type mOS struct { + waitsema uint32 // semaphore for parking on locks +} //go:noescape func thr_new(param *thrparam, size int32) int32 diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index e80d390e..8b3c4d0e 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -31,6 +31,12 @@ type mOS struct { // needPerThreadSyscall indicates that a per-thread syscall is required // for doAllThreadsSyscall. needPerThreadSyscall atomic.Uint8 + + // This is a pointer to a chunk of memory allocated with a special + // mmap invocation in vgetrandomGetState(). + vgetrandomState uintptr + + waitsema uint32 // semaphore for parking on locks } //go:noescape @@ -292,13 +298,19 @@ func sysargs(argc int32, argv **byte) { var secureMode bool func sysauxv(auxv []uintptr) (pairs int) { + // Process the auxiliary vector entries provided by the kernel when the + // program is executed. See getauxval(3). var i int for ; auxv[i] != _AT_NULL; i += 2 { tag, val := auxv[i], auxv[i+1] switch tag { case _AT_RANDOM: - // The kernel provides a pointer to 16-bytes - // worth of random data. + // The kernel provides a pointer to 16 bytes of cryptographically + // random data. Note that in cgo programs this value may have + // already been used by libc at this point, and in particular glibc + // and musl use the value as-is for stack and pointer protector + // cookies from libc_start_main and/or dl_start. Also, cgo programs + // may use the value after we do. startupRand = (*[16]byte)(unsafe.Pointer(val))[:] case _AT_PAGESZ: @@ -344,11 +356,14 @@ func osinit() { ncpu = getproccount() physHugePageSize = getHugePageSize() osArchInit() + vgetrandomInit() } var urandom_dev = []byte("/dev/urandom\x00") func readRandom(r []byte) int { + // Note that all supported Linux kernels should provide AT_RANDOM which + // populates startupRand, so this fallback should be unreachable. fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) closefd(fd) @@ -400,6 +415,10 @@ func unminit() { // Called from exitm, but not from drop, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. func mdestroy(mp *m) { + if mp.vgetrandomState != 0 { + vgetrandomPutState(mp.vgetrandomState) + mp.vgetrandomState = 0 + } } // #ifdef GOARCH_386 @@ -636,7 +655,7 @@ func setThreadCPUProfiler(hz int32) { // spend shows up as a 10% chance of one sample (for an expected value of // 0.1 samples), and so that "two and six tenths" periods of CPU spend show // up as a 60% chance of 3 samples and a 40% chance of 2 samples (for an - // expected value of 2.6). Set the initial delay to a value in the unifom + // expected value of 2.6). Set the initial delay to a value in the uniform // random distribution between 0 and the desired period. And because "0" // means "disable timer", add 1 so the half-open interval [0,period) turns // into (0,period]. diff --git a/src/runtime/os_linux_loong64.go b/src/runtime/os_linux_loong64.go index 61213dad..03926feb 100644 --- a/src/runtime/os_linux_loong64.go +++ b/src/runtime/os_linux_loong64.go @@ -6,6 +6,13 @@ package runtime -func archauxv(tag, val uintptr) {} +import "internal/cpu" + +func archauxv(tag, val uintptr) { + switch tag { + case _AT_HWCAP: + cpu.HWCap = uint(val) + } +} func osArchInit() {} diff --git a/src/runtime/os_openbsd.go b/src/runtime/os_openbsd.go index 9a21d6a8..574bfa8b 100644 --- a/src/runtime/os_openbsd.go +++ b/src/runtime/os_openbsd.go @@ -139,6 +139,9 @@ func osinit() { physPageSize = getPageSize() } +// TODO(#69781): set startupRand using the .openbsd.randomdata ELF section. +// See SPECS.randomdata. + var urandom_dev = []byte("/dev/urandom\x00") //go:nosplit diff --git a/src/runtime/os_solaris.go b/src/runtime/os_solaris.go index bc00698c..5f6163f1 100644 --- a/src/runtime/os_solaris.go +++ b/src/runtime/os_solaris.go @@ -4,7 +4,10 @@ package runtime -import "unsafe" +import ( + "internal/runtime/sys" + "unsafe" +) type mts struct { tv_sec int64 @@ -42,10 +45,10 @@ func sysvicall0(fn *libcFunc) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil // See comment in sys_darwin.go:libcCall } @@ -80,10 +83,10 @@ func sysvicall1Err(fn *libcFunc, a1 uintptr) (r1, err uintptr) { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -120,10 +123,10 @@ func sysvicall2Err(fn *libcFunc, a1, a2 uintptr) (uintptr, uintptr) { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -159,10 +162,10 @@ func sysvicall3Err(fn *libcFunc, a1, a2, a3 uintptr) (r1, err uintptr) { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -189,10 +192,10 @@ func sysvicall4(fn *libcFunc, a1, a2, a3, a4 uintptr) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -219,10 +222,10 @@ func sysvicall5(fn *libcFunc, a1, a2, a3, a4, a5 uintptr) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -249,10 +252,10 @@ func sysvicall6(fn *libcFunc, a1, a2, a3, a4, a5, a6 uintptr) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } diff --git a/src/runtime/os_wasip1.go b/src/runtime/os_wasip1.go index acac2b3f..ed4f646a 100644 --- a/src/runtime/os_wasip1.go +++ b/src/runtime/os_wasip1.go @@ -6,7 +6,10 @@ package runtime -import "unsafe" +import ( + "structs" + "unsafe" +) // GOARCH=wasm currently has 64 bits pointers, but the WebAssembly host expects // pointers to be 32 bits so we use this type alias to represent pointers in @@ -48,31 +51,31 @@ func exit(code int32) //go:wasmimport wasi_snapshot_preview1 args_get //go:noescape -func args_get(argv, argvBuf unsafe.Pointer) errno +func args_get(argv *uintptr32, argvBuf *byte) errno //go:wasmimport wasi_snapshot_preview1 args_sizes_get //go:noescape -func args_sizes_get(argc, argvBufLen unsafe.Pointer) errno +func args_sizes_get(argc, argvBufLen *size) errno //go:wasmimport wasi_snapshot_preview1 clock_time_get //go:noescape -func clock_time_get(clock_id clockid, precision timestamp, time unsafe.Pointer) errno +func clock_time_get(clock_id clockid, precision timestamp, time *timestamp) errno //go:wasmimport wasi_snapshot_preview1 environ_get //go:noescape -func environ_get(environ, environBuf unsafe.Pointer) errno +func environ_get(environ *uintptr32, environBuf *byte) errno //go:wasmimport wasi_snapshot_preview1 environ_sizes_get //go:noescape -func environ_sizes_get(environCount, environBufLen unsafe.Pointer) errno +func environ_sizes_get(environCount, environBufLen *size) errno //go:wasmimport wasi_snapshot_preview1 fd_write //go:noescape -func fd_write(fd int32, iovs unsafe.Pointer, iovsLen size, nwritten unsafe.Pointer) errno +func fd_write(fd int32, iovs unsafe.Pointer, iovsLen size, nwritten *size) errno //go:wasmimport wasi_snapshot_preview1 random_get //go:noescape -func random_get(buf unsafe.Pointer, bufLen size) errno +func random_get(buf *byte, bufLen size) errno type eventtype = uint8 @@ -99,6 +102,7 @@ type userdata = uint64 // struct size because errno is declared as a 32 bits type, so we declare the // error field as a plain uint16. type event struct { + _ structs.HostLayout userdata userdata error uint16 typ eventtype @@ -106,6 +110,7 @@ type event struct { } type eventFdReadwrite struct { + _ structs.HostLayout nbytes filesize flags eventrwflags } @@ -117,6 +122,7 @@ const ( ) type subscriptionClock struct { + _ structs.HostLayout id clockid timeout timestamp precision timestamp @@ -124,10 +130,12 @@ type subscriptionClock struct { } type subscriptionFdReadwrite struct { + _ structs.HostLayout fd int32 } type subscription struct { + _ structs.HostLayout userdata userdata u subscriptionUnion } @@ -148,7 +156,7 @@ func (u *subscriptionUnion) subscriptionFdReadwrite() *subscriptionFdReadwrite { //go:wasmimport wasi_snapshot_preview1 poll_oneoff //go:noescape -func poll_oneoff(in, out unsafe.Pointer, nsubscriptions size, nevents unsafe.Pointer) errno +func poll_oneoff(in *subscription, out *event, nsubscriptions size, nevents *size) errno func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { iov := iovec{ @@ -156,7 +164,7 @@ func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { bufLen: size(n), } var nwritten size - if fd_write(int32(fd), unsafe.Pointer(&iov), 1, unsafe.Pointer(&nwritten)) != 0 { + if fd_write(int32(fd), unsafe.Pointer(&iov), 1, &nwritten) != 0 { throw("fd_write failed") } return int32(nwritten) @@ -175,13 +183,13 @@ func usleep(usec uint32) { subscription.timeout = timestamp(usec) * 1e3 subscription.precision = 1e3 - if poll_oneoff(unsafe.Pointer(&in), unsafe.Pointer(&out), 1, unsafe.Pointer(&nevents)) != 0 { + if poll_oneoff(&in, &out, 1, &nevents) != 0 { throw("wasi_snapshot_preview1.poll_oneoff") } } func readRandom(r []byte) int { - if random_get(unsafe.Pointer(&r[0]), size(len(r))) != 0 { + if random_get(&r[0], size(len(r))) != 0 { return 0 } return len(r) @@ -191,7 +199,7 @@ func goenvs() { // arguments var argc size var argvBufLen size - if args_sizes_get(unsafe.Pointer(&argc), unsafe.Pointer(&argvBufLen)) != 0 { + if args_sizes_get(&argc, &argvBufLen) != 0 { throw("args_sizes_get failed") } @@ -199,7 +207,7 @@ func goenvs() { if argc > 0 { argv := make([]uintptr32, argc) argvBuf := make([]byte, argvBufLen) - if args_get(unsafe.Pointer(&argv[0]), unsafe.Pointer(&argvBuf[0])) != 0 { + if args_get(&argv[0], &argvBuf[0]) != 0 { throw("args_get failed") } @@ -216,7 +224,7 @@ func goenvs() { // environment var environCount size var environBufLen size - if environ_sizes_get(unsafe.Pointer(&environCount), unsafe.Pointer(&environBufLen)) != 0 { + if environ_sizes_get(&environCount, &environBufLen) != 0 { throw("environ_sizes_get failed") } @@ -224,7 +232,7 @@ func goenvs() { if environCount > 0 { environ := make([]uintptr32, environCount) environBuf := make([]byte, environBufLen) - if environ_get(unsafe.Pointer(&environ[0]), unsafe.Pointer(&environBuf[0])) != 0 { + if environ_get(&environ[0], &environBuf[0]) != 0 { throw("environ_get failed") } @@ -245,7 +253,7 @@ func walltime() (sec int64, nsec int32) { func walltime1() (sec int64, nsec int32) { var time timestamp - if clock_time_get(clockRealtime, 0, unsafe.Pointer(&time)) != 0 { + if clock_time_get(clockRealtime, 0, &time) != 0 { throw("clock_time_get failed") } return int64(time / 1000000000), int32(time % 1000000000) @@ -253,7 +261,7 @@ func walltime1() (sec int64, nsec int32) { func nanotime1() int64 { var time timestamp - if clock_time_get(clockMonotonic, 0, unsafe.Pointer(&time)) != 0 { + if clock_time_get(clockMonotonic, 0, &time) != 0 { throw("clock_time_get failed") } return int64(time) diff --git a/src/runtime/os_wasm.go b/src/runtime/os_wasm.go index fbafc319..8046caf4 100644 --- a/src/runtime/os_wasm.go +++ b/src/runtime/os_wasm.go @@ -13,6 +13,7 @@ func osinit() { // https://webassembly.github.io/spec/core/exec/runtime.html#memory-instances physPageSize = 64 * 1024 initBloc() + blocMax = uintptr(currentMemory()) * physPageSize // record the initial linear memory size ncpu = 1 getg().m.procid = 2 } @@ -96,7 +97,7 @@ func signame(sig uint32) string { } func crash() { - *(*int32)(nil) = 0 + abort() } func initsig(preinit bool) { @@ -109,10 +110,10 @@ func newosproc(mp *m) { throw("newosproc: not implemented") } +// Do nothing on WASM platform, always return EPIPE to caller. +// //go:linkname os_sigpipe os.sigpipe -func os_sigpipe() { - throw("too many writes on closed pipe") -} +func os_sigpipe() {} //go:linkname syscall_now syscall.now func syscall_now() (sec int64, nsec int32) { diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index c76df9da..c85fab7b 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -8,6 +8,7 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" + "internal/runtime/sys" "unsafe" ) @@ -1025,10 +1026,10 @@ func stdcall(fn stdFunction) uintptr { if mp.profilehz != 0 && mp.libcallsp == 0 { // leave pc/sp for cpu profiler mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() resetLibcall = true // See comment in sys_darwin.go:libcCall } asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall)) diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 98e96b12..3ffb3966 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -8,8 +8,8 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" + "internal/runtime/sys" "internal/stringslite" - "runtime/internal/sys" "unsafe" ) @@ -111,13 +111,13 @@ func panicCheck2(err string) { // //go:yeswritebarrierrec func goPanicIndex(x int, y int) { - panicCheck1(getcallerpc(), "index out of range") + panicCheck1(sys.GetCallerPC(), "index out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsIndex}) } //go:yeswritebarrierrec func goPanicIndexU(x uint, y int) { - panicCheck1(getcallerpc(), "index out of range") + panicCheck1(sys.GetCallerPC(), "index out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsIndex}) } @@ -125,25 +125,25 @@ func goPanicIndexU(x uint, y int) { // //go:yeswritebarrierrec func goPanicSliceAlen(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAlen}) } //go:yeswritebarrierrec func goPanicSliceAlenU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAlen}) } //go:yeswritebarrierrec func goPanicSliceAcap(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAcap}) } //go:yeswritebarrierrec func goPanicSliceAcapU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAcap}) } @@ -151,57 +151,57 @@ func goPanicSliceAcapU(x uint, y int) { // //go:yeswritebarrierrec func goPanicSliceB(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceB}) } //go:yeswritebarrierrec func goPanicSliceBU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceB}) } // failures in the comparisons for s[::x], 0 <= x <= y (y == len(s) or cap(s)) func goPanicSlice3Alen(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Alen}) } func goPanicSlice3AlenU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Alen}) } func goPanicSlice3Acap(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Acap}) } func goPanicSlice3AcapU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Acap}) } // failures in the comparisons for s[:x:y], 0 <= x <= y func goPanicSlice3B(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3B}) } func goPanicSlice3BU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3B}) } // failures in the comparisons for s[x:y:], 0 <= x <= y func goPanicSlice3C(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3C}) } func goPanicSlice3CU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3C}) } // failures in the conversion ([x]T)(s) or (*[x]T)(s), 0 <= x <= y, y == len(s) func goPanicSliceConvert(x int, y int) { - panicCheck1(getcallerpc(), "slice length too short to convert to array or pointer to array") + panicCheck1(sys.GetCallerPC(), "slice length too short to convert to array or pointer to array") panic(boundsError{x: int64(x), signed: true, y: y, code: boundsConvert}) } @@ -229,7 +229,7 @@ var shiftError = error(errorString("negative shift amount")) //go:yeswritebarrierrec func panicshift() { - panicCheck1(getcallerpc(), "negative shift amount") + panicCheck1(sys.GetCallerPC(), "negative shift amount") panic(shiftError) } @@ -280,11 +280,11 @@ func deferproc(fn func()) { d.link = gp._defer gp._defer = d d.fn = fn - d.pc = getcallerpc() - // We must not be preempted between calling getcallersp and - // storing it to d.sp because getcallersp's result is a + d.pc = sys.GetCallerPC() + // We must not be preempted between calling GetCallerSP and + // storing it to d.sp because GetCallerSP's result is a // uintptr stack pointer. - d.sp = getcallersp() + d.sp = sys.GetCallerSP() // deferproc returns 0 normally. // a deferred func that stops a panic @@ -394,11 +394,11 @@ func deferrangefunc() any { d := newdefer() d.link = gp._defer gp._defer = d - d.pc = getcallerpc() - // We must not be preempted between calling getcallersp and - // storing it to d.sp because getcallersp's result is a + d.pc = sys.GetCallerPC() + // We must not be preempted between calling GetCallerSP and + // storing it to d.sp because GetCallerSP's result is a // uintptr stack pointer. - d.sp = getcallersp() + d.sp = sys.GetCallerSP() d.rangefunc = true d.head = new(atomic.Pointer[_defer]) @@ -416,7 +416,7 @@ func badDefer() *_defer { func deferprocat(fn func(), frame any) { head := frame.(*atomic.Pointer[_defer]) if raceenabled { - racewritepc(unsafe.Pointer(head), getcallerpc(), abi.FuncPCABIInternal(deferprocat)) + racewritepc(unsafe.Pointer(head), sys.GetCallerPC(), abi.FuncPCABIInternal(deferprocat)) } d1 := newdefer() d1.fn = fn @@ -440,7 +440,7 @@ func deferprocat(fn func(), frame any) { func deferconvert(d0 *_defer) { head := d0.head if raceenabled { - racereadpc(unsafe.Pointer(head), getcallerpc(), abi.FuncPCABIInternal(deferconvert)) + racereadpc(unsafe.Pointer(head), sys.GetCallerPC(), abi.FuncPCABIInternal(deferconvert)) } tail := d0.link d0.rangefunc = false @@ -484,8 +484,8 @@ func deferprocStack(d *_defer) { // are initialized here. d.heap = false d.rangefunc = false - d.sp = getcallersp() - d.pc = getcallerpc() + d.sp = sys.GetCallerSP() + d.pc = sys.GetCallerPC() // The lines below implement: // d.panic = nil // d.fd = nil @@ -596,7 +596,7 @@ func deferreturn() { var p _panic p.deferreturn = true - p.start(getcallerpc(), unsafe.Pointer(getcallersp())) + p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP())) for { fn, ok := p.nextDefer() if !ok { @@ -614,13 +614,15 @@ func deferreturn() { // without func main returning. Since func main has not returned, // the program continues execution of other goroutines. // If all other goroutines exit, the program crashes. +// +// It crashes if called from a thread not created by the Go runtime. func Goexit() { // Create a panic object for Goexit, so we can recognize when it might be // bypassed by a recover(). var p _panic p.goexit = true - p.start(getcallerpc(), unsafe.Pointer(getcallersp())) + p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP())) for { fn, ok := p.nextDefer() if !ok { @@ -776,7 +778,7 @@ func gopanic(e any) { runningPanicDefers.Add(1) - p.start(getcallerpc(), unsafe.Pointer(getcallersp())) + p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP())) for { fn, ok := p.nextDefer() if !ok { @@ -815,8 +817,8 @@ func (p *_panic) start(pc uintptr, sp unsafe.Pointer) { // that have been recovered. Also, so that if p is from Goexit, we // can restart its defer processing loop if a recovered panic tries // to jump past it. - p.startPC = getcallerpc() - p.startSP = unsafe.Pointer(getcallersp()) + p.startPC = sys.GetCallerPC() + p.startSP = unsafe.Pointer(sys.GetCallerSP()) if p.deferreturn { p.sp = sp @@ -1031,14 +1033,41 @@ func sync_fatal(s string) { fatal(s) } +//go:linkname rand_fatal crypto/rand.fatal +func rand_fatal(s string) { + fatal(s) +} + +//go:linkname sysrand_fatal crypto/internal/sysrand.fatal +func sysrand_fatal(s string) { + fatal(s) +} + +//go:linkname fips_fatal crypto/internal/fips140.fatal +func fips_fatal(s string) { + fatal(s) +} + +//go:linkname maps_fatal internal/runtime/maps.fatal +func maps_fatal(s string) { + fatal(s) +} + +//go:linkname internal_sync_throw internal/sync.throw +func internal_sync_throw(s string) { + throw(s) +} + +//go:linkname internal_sync_fatal internal/sync.fatal +func internal_sync_fatal(s string) { + fatal(s) +} + // throw triggers a fatal error that dumps a stack trace and exits. // // throw should be used for runtime-internal fatal errors where Go itself, // rather than user code, may be at fault for the failure. // -// NOTE: temporarily marked "go:noinline" pending investigation/fix of -// issue #67274, so as to fix longtest builders. -// // throw should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: @@ -1079,6 +1108,7 @@ func throw(s string) { func fatal(s string) { // Everything fatal does should be recursively nosplit so it // can be called even when it's unsafe to grow the stack. + printlock() // Prevent multiple interleaved fatal reports. See issue 69447. systemstack(func() { print("fatal error: ") printindented(s) // logically printpanicval(s), but avoids convTstring write barrier @@ -1086,6 +1116,7 @@ func fatal(s string) { }) fatalthrow(throwTypeUser) + printunlock() } // runningPanicDefers is non-zero while running deferred functions for panic. @@ -1121,7 +1152,7 @@ func recovery(gp *g) { // frames that we've already processed. // // There's a similar issue with nested panics, when the inner - // panic supercedes the outer panic. Again, we end up needing to + // panic supersedes the outer panic. Again, we end up needing to // walk the same stack frames. // // These are probably pretty rare occurrences in practice, and @@ -1225,8 +1256,8 @@ func recovery(gp *g) { // //go:nosplit func fatalthrow(t throwType) { - pc := getcallerpc() - sp := getcallersp() + pc := sys.GetCallerPC() + sp := sys.GetCallerSP() gp := getg() if gp.m.throwing == throwTypeNone { @@ -1261,8 +1292,8 @@ func fatalthrow(t throwType) { // //go:nosplit func fatalpanic(msgs *_panic) { - pc := getcallerpc() - sp := getcallersp() + pc := sys.GetCallerPC() + sp := sys.GetCallerSP() gp := getg() var docrash bool // Switch to the system stack to avoid any stack growth, which diff --git a/src/runtime/panic32.go b/src/runtime/panic32.go index fa3f2bf2..cd34485a 100644 --- a/src/runtime/panic32.go +++ b/src/runtime/panic32.go @@ -6,82 +6,86 @@ package runtime +import ( + "internal/runtime/sys" +) + // Additional index/slice error paths for 32-bit platforms. // Used when the high word of a 64-bit index is not zero. // failures in the comparisons for s[x], 0 <= x < y (y == len(s)) func goPanicExtendIndex(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "index out of range") + panicCheck1(sys.GetCallerPC(), "index out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsIndex}) } func goPanicExtendIndexU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "index out of range") + panicCheck1(sys.GetCallerPC(), "index out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsIndex}) } // failures in the comparisons for s[:x], 0 <= x <= y (y == len(s) or cap(s)) func goPanicExtendSliceAlen(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSliceAlen}) } func goPanicExtendSliceAlenU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSliceAlen}) } func goPanicExtendSliceAcap(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSliceAcap}) } func goPanicExtendSliceAcapU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSliceAcap}) } // failures in the comparisons for s[x:y], 0 <= x <= y func goPanicExtendSliceB(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSliceB}) } func goPanicExtendSliceBU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSliceB}) } // failures in the comparisons for s[::x], 0 <= x <= y (y == len(s) or cap(s)) func goPanicExtendSlice3Alen(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3Alen}) } func goPanicExtendSlice3AlenU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3Alen}) } func goPanicExtendSlice3Acap(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3Acap}) } func goPanicExtendSlice3AcapU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3Acap}) } // failures in the comparisons for s[:x:y], 0 <= x <= y func goPanicExtendSlice3B(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3B}) } func goPanicExtendSlice3BU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3B}) } // failures in the comparisons for s[x:y:], 0 <= x <= y func goPanicExtendSlice3C(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3C}) } func goPanicExtendSlice3CU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3C}) } diff --git a/src/runtime/pinner.go b/src/runtime/pinner.go index 7a9c3815..543bfdb7 100644 --- a/src/runtime/pinner.go +++ b/src/runtime/pinner.go @@ -331,7 +331,7 @@ func (span *mspan) incPinCounter(offset uintptr) { rec = (*specialPinCounter)(mheap_.specialPinCounterAlloc.alloc()) unlock(&mheap_.speciallock) // splice in record, fill in offset. - rec.special.offset = uint16(offset) + rec.special.offset = offset rec.special.kind = _KindSpecialPinCounter rec.special.next = *ref *ref = (*special)(unsafe.Pointer(rec)) diff --git a/src/runtime/pprof/label.go b/src/runtime/pprof/label.go index 41eece2f..4c1d8d38 100644 --- a/src/runtime/pprof/label.go +++ b/src/runtime/pprof/label.go @@ -27,7 +27,7 @@ type labelContextKey struct{} func labelValue(ctx context.Context) labelMap { labels, _ := ctx.Value(labelContextKey{}).(*labelMap) if labels == nil { - return labelMap(nil) + return labelMap{} } return *labels } @@ -35,7 +35,9 @@ func labelValue(ctx context.Context) labelMap { // labelMap is the representation of the label set held in the context type. // This is an initial implementation, but it will be replaced with something // that admits incremental immutable modification more efficiently. -type labelMap map[string]string +type labelMap struct { + LabelSet +} // String satisfies Stringer and returns key, value pairs in a consistent // order. @@ -43,14 +45,13 @@ func (l *labelMap) String() string { if l == nil { return "" } - keyVals := make([]string, 0, len(*l)) + keyVals := make([]string, 0, len(l.list)) - for k, v := range *l { - keyVals = append(keyVals, fmt.Sprintf("%q:%q", k, v)) + for _, lbl := range l.list { + keyVals = append(keyVals, fmt.Sprintf("%q:%q", lbl.key, lbl.value)) } slices.Sort(keyVals) - return "{" + strings.Join(keyVals, ", ") + "}" } @@ -58,17 +59,38 @@ func (l *labelMap) String() string { // A label overwrites a prior label with the same key. func WithLabels(ctx context.Context, labels LabelSet) context.Context { parentLabels := labelValue(ctx) - childLabels := make(labelMap, len(parentLabels)) - // TODO(matloob): replace the map implementation with something - // more efficient so creating a child context WithLabels doesn't need - // to clone the map. - for k, v := range parentLabels { - childLabels[k] = v + return context.WithValue(ctx, labelContextKey{}, &labelMap{mergeLabelSets(parentLabels.LabelSet, labels)}) +} + +func mergeLabelSets(left, right LabelSet) LabelSet { + if len(left.list) == 0 { + return right + } else if len(right.list) == 0 { + return left } - for _, label := range labels.list { - childLabels[label.key] = label.value + + l, r := 0, 0 + result := make([]label, 0, len(right.list)) + for l < len(left.list) && r < len(right.list) { + switch strings.Compare(left.list[l].key, right.list[r].key) { + case -1: // left key < right key + result = append(result, left.list[l]) + l++ + case 1: // right key < left key + result = append(result, right.list[r]) + r++ + case 0: // keys are equal, right value overwrites left value + result = append(result, right.list[r]) + l++ + r++ + } } - return context.WithValue(ctx, labelContextKey{}, &childLabels) + + // Append the remaining elements + result = append(result, left.list[l:]...) + result = append(result, right.list[r:]...) + + return LabelSet{list: result} } // Labels takes an even number of strings representing key-value pairs @@ -82,8 +104,25 @@ func Labels(args ...string) LabelSet { panic("uneven number of arguments to pprof.Labels") } list := make([]label, 0, len(args)/2) + sortedNoDupes := true for i := 0; i+1 < len(args); i += 2 { list = append(list, label{key: args[i], value: args[i+1]}) + sortedNoDupes = sortedNoDupes && (i < 2 || args[i] > args[i-2]) + } + if !sortedNoDupes { + // slow path: keys are unsorted, contain duplicates, or both + slices.SortStableFunc(list, func(a, b label) int { + return strings.Compare(a.key, b.key) + }) + deduped := make([]label, 0, len(list)) + for i, lbl := range list { + if i == 0 || lbl.key != list[i-1].key { + deduped = append(deduped, lbl) + } else { + deduped[len(deduped)-1] = lbl + } + } + list = deduped } return LabelSet{list: list} } @@ -92,16 +131,20 @@ func Labels(args ...string) LabelSet { // whether that label exists. func Label(ctx context.Context, key string) (string, bool) { ctxLabels := labelValue(ctx) - v, ok := ctxLabels[key] - return v, ok + for _, lbl := range ctxLabels.list { + if lbl.key == key { + return lbl.value, true + } + } + return "", false } // ForLabels invokes f with each label set on the context. // The function f should return true to continue iteration or false to stop iteration early. func ForLabels(ctx context.Context, f func(key, value string) bool) { ctxLabels := labelValue(ctx) - for k, v := range ctxLabels { - if !f(k, v) { + for _, lbl := range ctxLabels.list { + if !f(lbl.key, lbl.value) { break } } diff --git a/src/runtime/pprof/label_test.go b/src/runtime/pprof/label_test.go index cefd9a53..3018693c 100644 --- a/src/runtime/pprof/label_test.go +++ b/src/runtime/pprof/label_test.go @@ -6,8 +6,10 @@ package pprof import ( "context" + "fmt" "reflect" - "sort" + "slices" + "strings" "testing" ) @@ -17,16 +19,10 @@ func labelsSorted(ctx context.Context) []label { ls = append(ls, label{key, value}) return true }) - sort.Sort(labelSorter(ls)) + slices.SortFunc(ls, func(a, b label) int { return strings.Compare(a.key, b.key) }) return ls } -type labelSorter []label - -func (s labelSorter) Len() int { return len(s) } -func (s labelSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s labelSorter) Less(i, j int) bool { return s[i].key < s[j].key } - func TestContextLabels(t *testing.T) { // Background context starts with no labels. ctx := context.Background() @@ -97,16 +93,18 @@ func TestLabelMapStringer(t *testing.T) { expected: "{}", }, { m: labelMap{ - "foo": "bar", + Labels("foo", "bar"), }, expected: `{"foo":"bar"}`, }, { m: labelMap{ - "foo": "bar", - "key1": "value1", - "key2": "value2", - "key3": "value3", - "key4WithNewline": "\nvalue4", + Labels( + "foo", "bar", + "key1", "value1", + "key2", "value2", + "key3", "value3", + "key4WithNewline", "\nvalue4", + ), }, expected: `{"foo":"bar", "key1":"value1", "key2":"value2", "key3":"value3", "key4WithNewline":"\nvalue4"}`, }, @@ -116,3 +114,75 @@ func TestLabelMapStringer(t *testing.T) { } } } + +func BenchmarkLabels(b *testing.B) { + b.Run("set-one", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(context.Background(), Labels("key", "value"), func(context.Context) {}) + } + }) + + b.Run("merge-one", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels("key1", "val1")) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(ctx, Labels("key2", "value2"), func(context.Context) {}) + } + }) + + b.Run("overwrite-one", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels("key", "val")) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(ctx, Labels("key", "value"), func(context.Context) {}) + } + }) + + for _, scenario := range []string{"ordered", "unordered"} { + var labels []string + for i := 0; i < 10; i++ { + labels = append(labels, fmt.Sprintf("key%03d", i), fmt.Sprintf("value%03d", i)) + } + if scenario == "unordered" { + labels[0], labels[len(labels)-1] = labels[len(labels)-1], labels[0] + } + + b.Run(scenario, func(b *testing.B) { + b.Run("set-many", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(context.Background(), Labels(labels...), func(context.Context) {}) + } + }) + + b.Run("merge-many", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels(labels[:len(labels)/2]...)) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + Do(ctx, Labels(labels[len(labels)/2:]...), func(context.Context) {}) + } + }) + + b.Run("overwrite-many", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels(labels...)) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(ctx, Labels(labels...), func(context.Context) {}) + } + }) + }) + } + + // TODO: hit slow path in Labels +} diff --git a/src/runtime/pprof/mprof_test.go b/src/runtime/pprof/mprof_test.go index ef373b36..7c4a37e3 100644 --- a/src/runtime/pprof/mprof_test.go +++ b/src/runtime/pprof/mprof_test.go @@ -9,6 +9,7 @@ package pprof import ( "bytes" "fmt" + "internal/asan" "internal/profile" "reflect" "regexp" @@ -63,6 +64,10 @@ func allocateReflect() { var memoryProfilerRun = 0 func TestMemoryProfiler(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan throw off the test; see #70079") + } + // Disable sampling, otherwise it's difficult to assert anything. oldRate := runtime.MemProfileRate runtime.MemProfileRate = 1 @@ -93,31 +98,31 @@ func TestMemoryProfiler(t *testing.T) { }{{ stk: []string{"runtime/pprof.allocatePersistent1K", "runtime/pprof.TestMemoryProfiler"}, legacy: fmt.Sprintf(`%v: %v \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocatePersistent1K\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:47 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:82 +# 0x[0-9,a-f]+ runtime/pprof\.allocatePersistent1K\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:48 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:87 `, 32*memoryProfilerRun, 1024*memoryProfilerRun, 32*memoryProfilerRun, 1024*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateTransient1M", "runtime/pprof.TestMemoryProfiler"}, legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient1M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:24 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:79 +# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient1M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:25 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:84 `, (1<<10)*memoryProfilerRun, (1<<20)*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateTransient2M", "runtime/pprof.TestMemoryProfiler"}, legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:30 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:80 +# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:31 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:85 `, memoryProfilerRun, (2<<20)*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateTransient2MInline", "runtime/pprof.TestMemoryProfiler"}, legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2MInline\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:34 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:81 +# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2MInline\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:35 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:86 `, memoryProfilerRun, (2<<20)*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateReflectTransient"}, legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @( 0x[0-9,a-f]+)+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateReflectTransient\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:55 +# 0x[0-9,a-f]+ runtime/pprof\.allocateReflectTransient\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:56 `, memoryProfilerRun, (2<<20)*memoryProfilerRun), }} diff --git a/src/runtime/pprof/pprof.go b/src/runtime/pprof/pprof.go index 4b7a9f63..b7680a13 100644 --- a/src/runtime/pprof/pprof.go +++ b/src/runtime/pprof/pprof.go @@ -44,7 +44,10 @@ // } // defer f.Close() // error handling omitted for example // runtime.GC() // get up-to-date statistics -// if err := pprof.WriteHeapProfile(f); err != nil { +// // Lookup("allocs") creates a profile similar to go test -memprofile. +// // Alternatively, use Lookup("heap") for a profile +// // that has inuse_space as the default index. +// if err := pprof.Lookup("allocs").WriteTo(f, 0); err != nil { // log.Fatal("could not write memory profile: ", err) // } // } @@ -513,8 +516,8 @@ func printCountProfile(w io.Writer, debug int, name string, p countProfile) erro var labels func() if p.Label(idx) != nil { labels = func() { - for k, v := range *p.Label(idx) { - b.pbLabel(tagSample_Label, k, v, 0) + for _, lbl := range p.Label(idx).list { + b.pbLabel(tagSample_Label, lbl.key, lbl.value, 0) } } } @@ -552,7 +555,7 @@ func printStackRecord(w io.Writer, stk []uintptr, allFrames bool) { if name == "" { show = true fmt.Fprintf(w, "#\t%#x\n", frame.PC) - } else if name != "runtime.goexit" && (show || !strings.HasPrefix(name, "runtime.")) { + } else if name != "runtime.goexit" && (show || !(strings.HasPrefix(name, "runtime.") || strings.HasPrefix(name, "internal/runtime/"))) { // Hide runtime.goexit and any runtime functions at the beginning. // This is useful mainly for allocation traces. show = true diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index da4ad17d..bba66ba4 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -22,6 +22,7 @@ import ( "regexp" "runtime" "runtime/debug" + "slices" "strconv" "strings" "sync" @@ -521,15 +522,6 @@ func diffCPUTime(t *testing.T, f func()) (user, system time.Duration) { return 0, 0 } -func contains(slice []string, s string) bool { - for i := range slice { - if slice[i] == s { - return true - } - } - return false -} - // stackContains matches if a function named spec appears anywhere in the stack trace. func stackContains(spec string, count uintptr, stk []*profile.Location, labels map[string][]string) bool { for _, loc := range stk { @@ -1237,7 +1229,7 @@ func blockFrequentShort(rate int) { } } -// blockFrequentShort produces 10000 block events with an average duration of +// blockInfrequentLong produces 10000 block events with an average duration of // rate. func blockInfrequentLong(rate int) { for i := 0; i < 10000; i++ { @@ -1410,7 +1402,7 @@ func TestMutexProfileRateAdjust(t *testing.T) { blockMutex(t) contentions, delay := readProfile() - if contentions == 0 || delay == 0 { + if contentions == 0 { // low-resolution timers can have delay of 0 in mutex profile t.Fatal("did not see expected function in profile") } runtime.SetMutexProfileFraction(0) @@ -1490,11 +1482,11 @@ func TestGoroutineCounts(t *testing.T) { goroutineProf.WriteTo(&w, 1) prof := w.String() - labels := labelMap{"label": "value"} + labels := labelMap{Labels("label", "value")} labelStr := "\n# labels: " + labels.String() - selfLabel := labelMap{"self-label": "self-value"} + selfLabel := labelMap{Labels("self-label", "self-value")} selfLabelStr := "\n# labels: " + selfLabel.String() - fingLabel := labelMap{"fing-label": "fing-value"} + fingLabel := labelMap{Labels("fing-label", "fing-value")} fingLabelStr := "\n# labels: " + fingLabel.String() orderedPrefix := []string{ "\n50 @ ", @@ -1960,7 +1952,7 @@ func stackContainsLabeled(spec string, count uintptr, stk []*profile.Location, l if !ok { panic("missing = in key/value spec") } - if !contains(labels[k], v) { + if !slices.Contains(labels[k], v) { return false } return stackContains(base, count, stk, labels) @@ -2077,7 +2069,7 @@ func TestLabelSystemstack(t *testing.T) { // * labelHog should always be labeled. // * The label should _only_ appear on labelHog and the Do call above. for _, s := range p.Sample { - isLabeled := s.Label != nil && contains(s.Label["key"], "value") + isLabeled := s.Label != nil && slices.Contains(s.Label["key"], "value") var ( mayBeLabeled bool mustBeLabeled string @@ -2553,19 +2545,34 @@ func TestProfilerStackDepth(t *testing.T) { t.Logf("Profile = %v", p) stks := profileStacks(p) - var stk []string - for _, s := range stks { - if hasPrefix(s, test.prefix) { - stk = s - break + var matchedStacks [][]string + for _, stk := range stks { + if !hasPrefix(stk, test.prefix) { + continue } + // We may get multiple stacks which contain the prefix we want, but + // which might not have enough frames, e.g. if the profiler hides + // some leaf frames that would count against the stack depth limit. + // Check for at least one match + matchedStacks = append(matchedStacks, stk) + if len(stk) != depth { + continue + } + if rootFn, wantFn := stk[depth-1], "runtime/pprof.produceProfileEvents"; rootFn != wantFn { + continue + } + // Found what we wanted + return } - if len(stk) != depth { - t.Fatalf("want stack depth = %d, got %d", depth, len(stk)) - } + for _, stk := range matchedStacks { + t.Logf("matched stack=%s", stk) + if len(stk) != depth { + t.Errorf("want stack depth = %d, got %d", depth, len(stk)) + } - if rootFn, wantFn := stk[depth-1], "runtime/pprof.produceProfileEvents"; rootFn != wantFn { - t.Fatalf("want stack stack root %s, got %v", wantFn, rootFn) + if rootFn, wantFn := stk[depth-1], "runtime/pprof.produceProfileEvents"; rootFn != wantFn { + t.Errorf("want stack stack root %s, got %v", wantFn, rootFn) + } } }) } diff --git a/src/runtime/pprof/proto.go b/src/runtime/pprof/proto.go index 5214374b..a664fdc6 100644 --- a/src/runtime/pprof/proto.go +++ b/src/runtime/pprof/proto.go @@ -367,8 +367,8 @@ func (b *profileBuilder) build() { var labels func() if e.tag != nil { labels = func() { - for k, v := range *(*labelMap)(e.tag) { - b.pbLabel(tagSample_Label, k, v, 0) + for _, lbl := range (*labelMap)(e.tag).list { + b.pbLabel(tagSample_Label, lbl.key, lbl.value, 0) } } } @@ -404,6 +404,7 @@ func (b *profileBuilder) appendLocsForStack(locs []uint64, stk []uintptr) (newLo b.deck.reset() // The last frame might be truncated. Recover lost inline frames. + origStk := stk stk = runtime_expandFinalInlineFrame(stk) for len(stk) > 0 { @@ -440,6 +441,9 @@ func (b *profileBuilder) appendLocsForStack(locs []uint64, stk []uintptr) (newLo // Even if stk was truncated due to the stack depth // limit, expandFinalInlineFrame above has already // fixed the truncation, ensuring it is long enough. + if len(l.pcs) > len(stk) { + panic(fmt.Sprintf("stack too short to match cached location; stk = %#x, l.pcs = %#x, original stk = %#x", stk, l.pcs, origStk)) + } stk = stk[len(l.pcs):] continue } diff --git a/src/runtime/pprof/protomem.go b/src/runtime/pprof/protomem.go index ab3550f4..72aad82b 100644 --- a/src/runtime/pprof/protomem.go +++ b/src/runtime/pprof/protomem.go @@ -36,7 +36,7 @@ func writeHeapProto(w io.Writer, p []profilerecord.MemProfileRecord, rate int64, // what appendLocsForStack expects. if hideRuntime { for i, addr := range stk { - if f := runtime.FuncForPC(addr); f != nil && strings.HasPrefix(f.Name(), "runtime.") { + if f := runtime.FuncForPC(addr); f != nil && (strings.HasPrefix(f.Name(), "runtime.") || strings.HasPrefix(f.Name(), "internal/runtime/")) { continue } // Found non-runtime. Show any runtime uses above it. diff --git a/src/runtime/pprof/protomem_test.go b/src/runtime/pprof/protomem_test.go index 8e9732a3..4d08e67d 100644 --- a/src/runtime/pprof/protomem_test.go +++ b/src/runtime/pprof/protomem_test.go @@ -7,6 +7,7 @@ package pprof import ( "bytes" "fmt" + "internal/asan" "internal/profile" "internal/profilerecord" "internal/testenv" @@ -117,8 +118,11 @@ func locationToStrings(loc *profile.Location, funcs []string) []string { return funcs } -// This is a regression test for https://go.dev/issue/64528 . +// This is a regression test for https://go.dev/issue/64528. func TestGenericsHashKeyInPprofBuilder(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan throw off the test; see #70079") + } previousRate := runtime.MemProfileRate runtime.MemProfileRate = 1 defer func() { @@ -178,6 +182,9 @@ func nonRecursiveGenericAllocFunction[CurrentOp any, OtherOp any](alloc bool) { } func TestGenericsInlineLocations(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan throw off the test; see #70079") + } if testenv.OptimizationOff() { t.Skip("skipping test with optimizations disabled") } @@ -222,3 +229,61 @@ func TestGenericsInlineLocations(t *testing.T) { t.Errorf("expected a location with at least 3 functions\n%s\ngot\n%s\n", expectedLocation, actual) } } + +func growMap() { + m := make(map[int]int) + for i := range 512 { + m[i] = i + } +} + +// Runtime frames are hidden in heap profiles. +// This is a regression test for https://go.dev/issue/71174. +func TestHeapRuntimeFrames(t *testing.T) { + previousRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = previousRate + }() + + growMap() + + runtime.GC() + buf := bytes.NewBuffer(nil) + if err := WriteHeapProfile(buf); err != nil { + t.Fatalf("writing profile: %v", err) + } + p, err := profile.Parse(buf) + if err != nil { + t.Fatalf("profile.Parse: %v", err) + } + + actual := profileToStrings(p) + + // We must see growMap at least once. + foundGrowMap := false + for _, l := range actual { + if !strings.Contains(l, "runtime/pprof.growMap") { + continue + } + foundGrowMap = true + + // Runtime frames like mapassign and map internals should be hidden. + if strings.Contains(l, "runtime.") { + t.Errorf("Sample got %s, want no runtime frames", l) + } + if strings.Contains(l, "internal/runtime/") { + t.Errorf("Sample got %s, want no runtime frames", l) + } + if strings.Contains(l, "runtime/internal/") { + t.Errorf("Sample got %s, want no runtime frames", l) + } + if strings.Contains(l, "mapassign") { // in case mapassign moves to a package not matching above paths. + t.Errorf("Sample got %s, want no mapassign frames", l) + } + } + + if !foundGrowMap { + t.Errorf("Profile got:\n%s\nwant sample in runtime/pprof.growMap", strings.Join(actual, "\n")) + } +} diff --git a/src/runtime/pprof/runtime_test.go b/src/runtime/pprof/runtime_test.go index 0dd5324b..353ed8a3 100644 --- a/src/runtime/pprof/runtime_test.go +++ b/src/runtime/pprof/runtime_test.go @@ -7,7 +7,7 @@ package pprof import ( "context" "fmt" - "reflect" + "maps" "testing" ) @@ -15,11 +15,11 @@ func TestSetGoroutineLabels(t *testing.T) { sync := make(chan struct{}) wantLabels := map[string]string{} - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("Expected parent goroutine's profile labels to be empty before test, got %v", gotLabels) } go func() { - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("Expected child goroutine's profile labels to be empty before test, got %v", gotLabels) } sync <- struct{}{} @@ -29,11 +29,11 @@ func TestSetGoroutineLabels(t *testing.T) { wantLabels = map[string]string{"key": "value"} ctx := WithLabels(context.Background(), Labels("key", "value")) SetGoroutineLabels(ctx) - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("parent goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) } go func() { - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("child goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) } sync <- struct{}{} @@ -43,11 +43,11 @@ func TestSetGoroutineLabels(t *testing.T) { wantLabels = map[string]string{} ctx = context.Background() SetGoroutineLabels(ctx) - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("Expected parent goroutine's profile labels to be empty, got %v", gotLabels) } go func() { - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("Expected child goroutine's profile labels to be empty, got %v", gotLabels) } sync <- struct{}{} @@ -57,20 +57,20 @@ func TestSetGoroutineLabels(t *testing.T) { func TestDo(t *testing.T) { wantLabels := map[string]string{} - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("Expected parent goroutine's profile labels to be empty before Do, got %v", gotLabels) } Do(context.Background(), Labels("key1", "value1", "key2", "value2"), func(ctx context.Context) { wantLabels := map[string]string{"key1": "value1", "key2": "value2"} - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("parent goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) } sync := make(chan struct{}) go func() { wantLabels := map[string]string{"key1": "value1", "key2": "value2"} - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { t.Errorf("child goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) } sync <- struct{}{} @@ -80,7 +80,7 @@ func TestDo(t *testing.T) { }) wantLabels = map[string]string{} - if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + if gotLabels := getProfLabel(); !maps.Equal(gotLabels, wantLabels) { fmt.Printf("%#v", gotLabels) fmt.Printf("%#v", wantLabels) t.Errorf("Expected parent goroutine's profile labels to be empty after Do, got %v", gotLabels) @@ -92,5 +92,9 @@ func getProfLabel() map[string]string { if l == nil { return map[string]string{} } - return *l + m := make(map[string]string, len(l.list)) + for _, lbl := range l.list { + m[lbl.key] = lbl.value + } + return m } diff --git a/src/runtime/pprof/vminfo_darwin_test.go b/src/runtime/pprof/vminfo_darwin_test.go index 64158720..6d375c5d 100644 --- a/src/runtime/pprof/vminfo_darwin_test.go +++ b/src/runtime/pprof/vminfo_darwin_test.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "testing" + "time" ) func TestVMInfo(t *testing.T) { @@ -56,18 +57,35 @@ func TestVMInfo(t *testing.T) { } } +type mapping struct { + hi, lo uint64 + err error +} + func useVMMapWithRetry(t *testing.T) (hi, lo uint64, err error) { var retryable bool - for { - hi, lo, retryable, err = useVMMap(t) - if err == nil { - return hi, lo, nil + ch := make(chan mapping) + go func() { + for { + hi, lo, retryable, err = useVMMap(t) + if err == nil { + ch <- mapping{hi, lo, nil} + return + } + if !retryable { + ch <- mapping{0, 0, err} + return + } + t.Logf("retrying vmmap after error: %v", err) } - if !retryable { - return 0, 0, err - } - t.Logf("retrying vmmap after error: %v", err) + }() + select { + case m := <-ch: + return m.hi, m.lo, m.err + case <-time.After(time.Minute): + t.Skip("vmmap taking too long") } + return 0, 0, fmt.Errorf("unreachable") } func useVMMap(t *testing.T) (hi, lo uint64, retryable bool, err error) { @@ -79,7 +97,7 @@ func useVMMap(t *testing.T) (hi, lo uint64, retryable bool, err error) { t.Logf("vmmap output: %s", out) if ee, ok := cmdErr.(*exec.ExitError); ok && len(ee.Stderr) > 0 { t.Logf("%v: %v\n%s", cmd, cmdErr, ee.Stderr) - if testing.Short() && strings.Contains(string(ee.Stderr), "No process corpse slots currently available, waiting to get one") { + if testing.Short() && (strings.Contains(string(ee.Stderr), "No process corpse slots currently available, waiting to get one") || strings.Contains(string(ee.Stderr), "Failed to generate corpse from the process")) { t.Skipf("Skipping knwn flake in short test mode") } retryable = bytes.Contains(ee.Stderr, []byte("resource shortage")) diff --git a/src/runtime/preempt_amd64.s b/src/runtime/preempt_amd64.s index 94a84fb7..8e3ed0d7 100644 --- a/src/runtime/preempt_amd64.s +++ b/src/runtime/preempt_amd64.s @@ -27,13 +27,6 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVQ R13, 88(SP) MOVQ R14, 96(SP) MOVQ R15, 104(SP) - #ifdef GOOS_darwin - #ifndef hasAVX - CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0 - JE 2(PC) - #endif - VZEROUPPER - #endif MOVUPS X0, 112(SP) MOVUPS X1, 128(SP) MOVUPS X2, 144(SP) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index e3cdf719..e9873e54 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -11,8 +11,8 @@ import ( "internal/goos" "internal/runtime/atomic" "internal/runtime/exithook" + "internal/runtime/sys" "internal/stringslite" - "runtime/internal/sys" "unsafe" ) @@ -266,6 +266,17 @@ func main() { if isarchive || islibrary { // A program compiled with -buildmode=c-archive or c-shared // has a main, but it is not executed. + if GOARCH == "wasm" { + // On Wasm, pause makes it return to the host. + // Unlike cgo callbacks where Ms are created on demand, + // on Wasm we have only one M. So we keep this M (and this + // G) for callbacks. + // Using the caller's SP unwinds this frame and backs to + // goexit. The -16 is: 8 for goexit's (fake) return PC, + // and pause's epilogue pops 8. + pause(sys.GetCallerSP() - 16) // should not return + panic("unreachable") + } return } fn := main_main // make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime @@ -739,6 +750,11 @@ func cpuinit(env string) { case "arm64": arm64HasATOMICS = cpu.ARM64.HasATOMICS + + case "loong64": + loong64HasLAMCAS = cpu.Loong64.HasLAMCAS + loong64HasLAM_BH = cpu.Loong64.HasLAM_BH + loong64HasLSX = cpu.Loong64.HasLSX } } @@ -799,6 +815,8 @@ func schedinit() { // extremely short. lockInit(&memstats.heapStats.noPLock, lockRankLeafRank) + lockVerifyMSize() + // raceinit must be the first call to race detector. // In particular, it must be done before mallocinit below calls racemapshadow. gp := getg() @@ -1211,6 +1229,12 @@ func casgstatus(gp *g, oldval, newval uint32) { } } + if gp.syncGroup != nil { + systemstack(func() { + gp.syncGroup.changegstatus(gp, oldval, newval) + }) + } + if oldval == _Grunning { // Track every gTrackingPeriod time a goroutine transitions out of running. if casgstatusAlwaysTrack || gp.trackingSeq%gTrackingPeriod == 0 { @@ -1293,25 +1317,6 @@ func casGToWaitingForGC(gp *g, old uint32, reason waitReason) { casGToWaiting(gp, old, reason) } -// casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable. -// Returns old status. Cannot call casgstatus directly, because we are racing with an -// async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus, -// it might have become Grunnable by the time we get to the cas. If we called casgstatus, -// it would loop waiting for the status to go back to Gwaiting, which it never will. -// -//go:nosplit -func casgcopystack(gp *g) uint32 { - for { - oldstatus := readgstatus(gp) &^ _Gscan - if oldstatus != _Gwaiting && oldstatus != _Grunnable { - throw("copystack: bad status, not Gwaiting or Grunnable") - } - if gp.atomicstatus.CompareAndSwap(oldstatus, _Gcopystack) { - return oldstatus - } - } -} - // casGToPreemptScan transitions gp from _Grunning to _Gscan|_Gpreempted. // // TODO(austin): This is the only status operation that both changes @@ -1323,6 +1328,12 @@ func casGToPreemptScan(gp *g, old, new uint32) { acquireLockRankAndM(lockRankGscan) for !gp.atomicstatus.CompareAndSwap(_Grunning, _Gscan|_Gpreempted) { } + // We never notify gp.syncGroup that the goroutine state has moved + // from _Grunning to _Gpreempted. We call syncGroup.changegstatus + // after status changes happen, but doing so here would violate the + // ordering between the gscan and synctest locks. syncGroup doesn't + // distinguish between _Grunning and _Gpreempted anyway, so not + // notifying it is fine. } // casGFromPreempted attempts to transition gp from _Gpreempted to @@ -1333,7 +1344,13 @@ func casGFromPreempted(gp *g, old, new uint32) bool { throw("bad g transition") } gp.waitreason = waitReasonPreempted - return gp.atomicstatus.CompareAndSwap(_Gpreempted, _Gwaiting) + if !gp.atomicstatus.CompareAndSwap(_Gpreempted, _Gwaiting) { + return false + } + if sg := gp.syncGroup; sg != nil { + sg.changegstatus(gp, _Gpreempted, _Gwaiting) + } + return true } // stwReason is an enumeration of reasons the world is stopping. @@ -1800,7 +1817,7 @@ func mstart0() { mexit(osStack) } -// The go:noinline is to guarantee the getcallerpc/getcallersp below are safe, +// The go:noinline is to guarantee the sys.GetCallerPC/sys.GetCallerSP below are safe, // so that we can set up g0.sched to return to the call of mstart1 above. // //go:noinline @@ -1818,8 +1835,8 @@ func mstart1() { // And goexit0 does a gogo that needs to return from mstart1 // and let mstart0 exit the thread. gp.sched.g = guintptr(unsafe.Pointer(gp)) - gp.sched.pc = getcallerpc() - gp.sched.sp = getcallersp() + gp.sched.pc = sys.GetCallerPC() + gp.sched.sp = sys.GetCallerSP() asminit() minit() @@ -1830,6 +1847,10 @@ func mstart1() { mstartm0() } + if debug.dataindependenttiming == 1 { + sys.EnableDIT() + } + if fn := gp.m.mstartfn; fn != nil { fn() } @@ -2318,7 +2339,7 @@ func needm(signal bool) { // Install g (= m->g0) and set the stack bounds // to match the current stack. setg(mp.g0) - sp := getcallersp() + sp := sys.GetCallerSP() callbackUpdateSystemStack(mp, sp, signal) // Should mark we are already in Go now. @@ -4071,6 +4092,15 @@ func park_m(gp *g) { trace := traceAcquire() + // If g is in a synctest group, we don't want to let the group + // become idle until after the waitunlockf (if any) has confirmed + // that the park is happening. + // We need to record gp.syncGroup here, since waitunlockf can change it. + sg := gp.syncGroup + if sg != nil { + sg.incActive() + } + if trace.ok() { // Trace the event before the transition. It may take a // stack trace, but we won't own the stack after the @@ -4093,6 +4123,9 @@ func park_m(gp *g) { if !ok { trace := traceAcquire() casgstatus(gp, _Gwaiting, _Grunnable) + if sg != nil { + sg.decActive() + } if trace.ok() { trace.GoUnpark(gp, 2) traceRelease(trace) @@ -4100,6 +4133,11 @@ func park_m(gp *g) { execute(gp, true) // Schedule it back, never returns. } } + + if sg != nil { + sg.decActive() + } + schedule() } @@ -4253,6 +4291,9 @@ func goyield_m(gp *g) { // Finishes execution of the current goroutine. func goexit1() { if raceenabled { + if gp := getg(); gp.syncGroup != nil { + racereleasemergeg(gp, gp.syncGroup.raceaddr()) + } racegoend() } trace := traceAcquire() @@ -4291,6 +4332,7 @@ func gdestroy(gp *g) { gp.param = nil gp.labels = nil gp.timer = nil + gp.syncGroup = nil if gcBlackenEnabled != 0 && gp.gcAssistBytes > 0 { // Flush assist credit to the global pool. This gives @@ -4311,6 +4353,9 @@ func gdestroy(gp *g) { if locked && mp.lockedInt != 0 { print("runtime: mp.lockedInt = ", mp.lockedInt, "\n") + if mp.isextra { + throw("runtime.Goexit called in a thread that was not created by the Go runtime") + } throw("exited a goroutine internally locked to the OS thread") } gfput(pp, gp) @@ -4483,7 +4528,7 @@ func entersyscall() { // the stack. This results in exceeding the nosplit stack requirements // on some platforms. fp := getcallerfp() - reentersyscall(getcallerpc(), getcallersp(), fp) + reentersyscall(sys.GetCallerPC(), sys.GetCallerSP(), fp) } func entersyscall_sysmon() { @@ -4548,8 +4593,8 @@ func entersyscallblock() { gp.m.p.ptr().syscalltick++ // Leave SP around for GC and traceback. - pc := getcallerpc() - sp := getcallersp() + pc := sys.GetCallerPC() + sp := sys.GetCallerSP() bp := getcallerfp() save(pc, sp, bp) gp.syscallsp = gp.sched.sp @@ -4581,7 +4626,7 @@ func entersyscallblock() { systemstack(entersyscallblock_handoff) // Resave for traceback during blocked call. - save(getcallerpc(), getcallersp(), getcallerfp()) + save(sys.GetCallerPC(), sys.GetCallerSP(), getcallerfp()) gp.m.locks-- } @@ -4619,7 +4664,7 @@ func exitsyscall() { gp := getg() gp.m.locks++ // see comment in entersyscall - if getcallersp() > gp.syscallsp { + if sys.GetCallerSP() > gp.syscallsp { throw("exitsyscall: syscall frame is no longer valid") } @@ -4835,7 +4880,6 @@ func exitsyscall0(gp *g) { // syscall_runtime_BeforeFork is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/containerd/containerd // - gvisor.dev/gvisor // // Do not remove or change the type signature. @@ -4865,7 +4909,6 @@ func syscall_runtime_BeforeFork() { // syscall_runtime_AfterFork is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/containerd/containerd // - gvisor.dev/gvisor // // Do not remove or change the type signature. @@ -4899,7 +4942,6 @@ var inForkedChild bool // syscall_runtime_AfterForkInChild is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/containerd/containerd // - gvisor.dev/gvisor // // Do not remove or change the type signature. @@ -4974,7 +5016,7 @@ func malg(stacksize int32) *g { // The compiler turns a go statement into a call to this. func newproc(fn *funcval) { gp := getg() - pc := getcallerpc() + pc := sys.GetCallerPC() systemstack(func() { newg := newproc1(fn, gp, pc, false, waitReasonZero) @@ -5037,7 +5079,8 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso if isSystemGoroutine(newg, false) { sched.ngsys.Add(1) } else { - // Only user goroutines inherit pprof labels. + // Only user goroutines inherit synctest groups and pprof labels. + newg.syncGroup = callergp.syncGroup if mp.curg != nil { newg.labels = mp.curg.labels } @@ -5064,7 +5107,6 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso status = _Gwaiting newg.waitreason = waitreason } - casgstatus(newg, _Gdead, status) if pp.goidcache == pp.goidcacheend { // Sched.goidgen is the last allocated id, // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. @@ -5074,6 +5116,7 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso pp.goidcacheend = pp.goidcache + _GoidCacheBatch } newg.goid = pp.goidcache + casgstatus(newg, _Gdead, status) pp.goidcache++ newg.trace.reset() if trace.ok() { @@ -5926,7 +5969,9 @@ func checkdead() { // For -buildmode=c-shared or -buildmode=c-archive it's OK if // there are no running goroutines. The calling program is // assumed to be running. - if islibrary || isarchive { + // One exception is Wasm, which is single-threaded. If we are + // in Go and all goroutines are blocked, it deadlocks. + if (islibrary || isarchive) && GOARCH != "wasm" { return } @@ -7133,6 +7178,31 @@ func sync_atomic_runtime_procUnpin() { procUnpin() } +// Active spinning for sync.Mutex. +// +//go:linkname internal_sync_runtime_canSpin internal/sync.runtime_canSpin +//go:nosplit +func internal_sync_runtime_canSpin(i int) bool { + // sync.Mutex is cooperative, so we are conservative with spinning. + // Spin only few times and only if running on a multicore machine and + // GOMAXPROCS>1 and there is at least one other running P and local runq is empty. + // As opposed to runtime mutex we don't do passive spinning here, + // because there can be work on global runq or on other Ps. + if i >= active_spin || ncpu <= 1 || gomaxprocs <= sched.npidle.Load()+sched.nmspinning.Load()+1 { + return false + } + if p := getg().m.p.ptr(); !runqempty(p) { + return false + } + return true +} + +//go:linkname internal_sync_runtime_doSpin internal/sync.runtime_doSpin +//go:nosplit +func internal_sync_runtime_doSpin() { + procyield(active_spin_cnt) +} + // Active spinning for sync.Mutex. // // sync_runtime_canSpin should be an internal detail, @@ -7148,18 +7218,7 @@ func sync_atomic_runtime_procUnpin() { //go:linkname sync_runtime_canSpin sync.runtime_canSpin //go:nosplit func sync_runtime_canSpin(i int) bool { - // sync.Mutex is cooperative, so we are conservative with spinning. - // Spin only few times and only if running on a multicore machine and - // GOMAXPROCS>1 and there is at least one other running P and local runq is empty. - // As opposed to runtime mutex we don't do passive spinning here, - // because there can be work on global runq or on other Ps. - if i >= active_spin || ncpu <= 1 || gomaxprocs <= sched.npidle.Load()+sched.nmspinning.Load()+1 { - return false - } - if p := getg().m.p.ptr(); !runqempty(p) { - return false - } - return true + return internal_sync_runtime_canSpin(i) } // sync_runtime_doSpin should be an internal detail, @@ -7175,7 +7234,7 @@ func sync_runtime_canSpin(i int) bool { //go:linkname sync_runtime_doSpin sync.runtime_doSpin //go:nosplit func sync_runtime_doSpin() { - procyield(active_spin_cnt) + internal_sync_runtime_doSpin() } var stealOrder randomOrder diff --git a/src/runtime/profbuf_test.go b/src/runtime/profbuf_test.go index dac78ffd..9050d1fa 100644 --- a/src/runtime/profbuf_test.go +++ b/src/runtime/profbuf_test.go @@ -5,8 +5,8 @@ package runtime_test import ( - "reflect" . "runtime" + "slices" "testing" "time" "unsafe" @@ -20,7 +20,7 @@ func TestProfBuf(t *testing.T) { } read := func(t *testing.T, b *ProfBuf, data []uint64, tags []unsafe.Pointer) { rdata, rtags, eof := b.Read(ProfBufNonBlocking) - if !reflect.DeepEqual(rdata, data) || !reflect.DeepEqual(rtags, tags) { + if !slices.Equal(rdata, data) || !slices.Equal(rtags, tags) { t.Fatalf("unexpected profile read:\nhave data %#x\nwant data %#x\nhave tags %#x\nwant tags %#x", rdata, data, rtags, tags) } if eof { @@ -32,7 +32,7 @@ func TestProfBuf(t *testing.T) { go func() { eof := data == nil rdata, rtags, reof := b.Read(ProfBufBlocking) - if !reflect.DeepEqual(rdata, data) || !reflect.DeepEqual(rtags, tags) || reof != eof { + if !slices.Equal(rdata, data) || !slices.Equal(rtags, tags) || reof != eof { // Errorf, not Fatalf, because called in goroutine. t.Errorf("unexpected profile read:\nhave data %#x\nwant data %#x\nhave tags %#x\nwant tags %#x\nhave eof=%v, want %v", rdata, data, rtags, tags, reof, eof) } diff --git a/src/runtime/proflabel.go b/src/runtime/proflabel.go index 1a5e7e5e..f9b9dd16 100644 --- a/src/runtime/proflabel.go +++ b/src/runtime/proflabel.go @@ -12,7 +12,6 @@ var labelSync uintptr // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/cloudwego/localsession -// - github.com/DataDog/datadog-agent // // Do not remove or change the type signature. // See go.dev/issue/67401. @@ -47,7 +46,6 @@ func runtime_setProfLabel(labels unsafe.Pointer) { // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/cloudwego/localsession -// - github.com/DataDog/datadog-agent // // Do not remove or change the type signature. // See go.dev/issue/67401. diff --git a/src/runtime/race.go b/src/runtime/race.go index 7d5cbce4..6b7bbe52 100644 --- a/src/runtime/race.go +++ b/src/runtime/race.go @@ -14,16 +14,49 @@ import ( // Public race detection API, present iff build with -race. func RaceRead(addr unsafe.Pointer) + +//go:linkname race_Read internal/race.Read +//go:nosplit +func race_Read(addr unsafe.Pointer) { + RaceRead(addr) +} + func RaceWrite(addr unsafe.Pointer) + +//go:linkname race_Write internal/race.Write +//go:nosplit +func race_Write(addr unsafe.Pointer) { + RaceWrite(addr) +} + func RaceReadRange(addr unsafe.Pointer, len int) + +//go:linkname race_ReadRange internal/race.ReadRange +//go:nosplit +func race_ReadRange(addr unsafe.Pointer, len int) { + RaceReadRange(addr, len) +} + func RaceWriteRange(addr unsafe.Pointer, len int) +//go:linkname race_WriteRange internal/race.WriteRange +//go:nosplit +func race_WriteRange(addr unsafe.Pointer, len int) { + RaceWriteRange(addr, len) +} + func RaceErrors() int { var n uint64 racecall(&__tsan_report_count, uintptr(unsafe.Pointer(&n)), 0, 0, 0) return int(n) } +//go:linkname race_Errors internal/race.Errors +//go:nosplit +func race_Errors() int { + return RaceErrors() +} + // RaceAcquire/RaceRelease/RaceReleaseMerge establish happens-before relations // between goroutines. These inform the race detector about actual synchronization // that it can't see for some reason (e.g. synchronization within RaceDisable/RaceEnable @@ -38,6 +71,12 @@ func RaceAcquire(addr unsafe.Pointer) { raceacquire(addr) } +//go:linkname race_Acquire internal/race.Acquire +//go:nosplit +func race_Acquire(addr unsafe.Pointer) { + RaceAcquire(addr) +} + // RaceRelease performs a release operation on addr that // can synchronize with a later RaceAcquire on addr. // @@ -49,6 +88,12 @@ func RaceRelease(addr unsafe.Pointer) { racerelease(addr) } +//go:linkname race_Release internal/race.Release +//go:nosplit +func race_Release(addr unsafe.Pointer) { + RaceRelease(addr) +} + // RaceReleaseMerge is like RaceRelease, but also establishes a happens-before // relation with the preceding RaceRelease or RaceReleaseMerge on addr. // @@ -60,6 +105,12 @@ func RaceReleaseMerge(addr unsafe.Pointer) { racereleasemerge(addr) } +//go:linkname race_ReleaseMerge internal/race.ReleaseMerge +//go:nosplit +func race_ReleaseMerge(addr unsafe.Pointer) { + RaceReleaseMerge(addr) +} + // RaceDisable disables handling of race synchronization events in the current goroutine. // Handling is re-enabled with RaceEnable. RaceDisable/RaceEnable can be nested. // Non-synchronization events (memory accesses, function entry/exit) still affect @@ -74,6 +125,12 @@ func RaceDisable() { gp.raceignore++ } +//go:linkname race_Disable internal/race.Disable +//go:nosplit +func race_Disable() { + RaceDisable() +} + // RaceEnable re-enables handling of race events in the current goroutine. // //go:nosplit @@ -85,6 +142,12 @@ func RaceEnable() { } } +//go:linkname race_Enable internal/race.Enable +//go:nosplit +func race_Enable() { + RaceEnable() +} + // Private interface for the runtime. const raceenabled = true @@ -105,6 +168,11 @@ func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { } } +//go:linkname race_ReadObjectPC internal/race.ReadObjectPC +func race_ReadObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { + raceReadObjectPC(t, addr, callerpc, pc) +} + func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { kind := t.Kind_ & abi.KindMask if kind == abi.Array || kind == abi.Struct { @@ -118,12 +186,27 @@ func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { } } +//go:linkname race_WriteObjectPC internal/race.WriteObjectPC +func race_WriteObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { + raceWriteObjectPC(t, addr, callerpc, pc) +} + //go:noescape func racereadpc(addr unsafe.Pointer, callpc, pc uintptr) //go:noescape func racewritepc(addr unsafe.Pointer, callpc, pc uintptr) +//go:linkname race_ReadPC internal/race.ReadPC +func race_ReadPC(addr unsafe.Pointer, callerpc, pc uintptr) { + racereadpc(addr, callerpc, pc) +} + +//go:linkname race_WritePC internal/race.WritePC +func race_WritePC(addr unsafe.Pointer, callerpc, pc uintptr) { + racewritepc(addr, callerpc, pc) +} + type symbolizeCodeContext struct { pc uintptr fn *byte diff --git a/src/runtime/race/output_test.go b/src/runtime/race/output_test.go index 0ee0f413..0d5c9096 100644 --- a/src/runtime/race/output_test.go +++ b/src/runtime/race/output_test.go @@ -476,5 +476,59 @@ Previous write at 0x[0-9,a-f]+ by main goroutine: main\.main\(\) .*/main.go:10 \+0x[0-9,a-f]+ +`}}, + {"non_inline_array_compare", "run", "", "atexit_sleep_ms=0", ` +package main + +import ( + "math/rand/v2" +) + +var x = [1024]byte{} + +var ch = make(chan bool) + +func main() { + started := make(chan struct{}) + go func() { + close(started) + var y = [len(x)]byte{} + eq := x == y + ch <- eq + }() + <-started + x[rand.IntN(len(x))]++ + println(<-ch) +} +`, []string{`================== +WARNING: DATA RACE +`}}, + {"non_inline_struct_compare", "run", "", "atexit_sleep_ms=0", ` +package main + +import "math/rand/v2" + +type S struct { + a [1024]byte +} + +var x = S{a: [1024]byte{}} + +var ch = make(chan bool) + +func main() { + started := make(chan struct{}) + go func() { + close(started) + var y = S{a: [len(x.a)]byte{}} + eq := x == y + ch <- eq + }() + <-started + x.a[rand.IntN(len(x.a))]++ + println(<-ch) +} +`, []string{`================== +WARNING: DATA RACE `}}, } diff --git a/src/runtime/race/race_test.go b/src/runtime/race/race_test.go index 4fe61683..cbc90ea0 100644 --- a/src/runtime/race/race_test.go +++ b/src/runtime/race/race_test.go @@ -107,6 +107,11 @@ func processLog(testName string, tsanLog []string) string { gotRace = true break } + if strings.Contains(s, "fatal error: concurrent map") { + // Detected by the runtime, not the race detector. + gotRace = true + break + } } failing := strings.Contains(testName, "Failing") @@ -177,8 +182,11 @@ func runTests(t *testing.T) ([]byte, error) { ) // There are races: we expect tests to fail and the exit code to be non-zero. out, _ := cmd.CombinedOutput() - if bytes.Contains(out, []byte("fatal error:")) { - // But don't expect runtime to crash. + fatals := bytes.Count(out, []byte("fatal error:")) + mapFatals := bytes.Count(out, []byte("fatal error: concurrent map")) + if fatals > mapFatals { + // But don't expect runtime to crash (other than + // in the map concurrent access detector). return out, fmt.Errorf("runtime fatal error") } return out, nil diff --git a/src/runtime/race/sched_test.go b/src/runtime/race/sched_test.go index a66860cd..edff0d5c 100644 --- a/src/runtime/race/sched_test.go +++ b/src/runtime/race/sched_test.go @@ -8,8 +8,8 @@ package race_test import ( "fmt" - "reflect" "runtime" + "slices" "strings" "testing" ) @@ -35,7 +35,7 @@ func TestRandomScheduling(t *testing.T) { } for i := 0; i < N; i++ { - if !reflect.DeepEqual(out[0], out[i]) { + if !slices.Equal(out[0], out[i]) { return // found a different order } } diff --git a/src/runtime/race/testdata/map_test.go b/src/runtime/race/testdata/map_test.go index 88e735ec..83f59b75 100644 --- a/src/runtime/race/testdata/map_test.go +++ b/src/runtime/race/testdata/map_test.go @@ -242,7 +242,7 @@ func TestRaceMapAssignMultipleReturn(t *testing.T) { } // BigKey and BigVal must be larger than 256 bytes, -// so that compiler sets KindGCProg for them. +// so that compiler stores them indirectly. type BigKey [1000]*int type BigVal struct { diff --git a/src/runtime/race/testdata/mop_test.go b/src/runtime/race/testdata/mop_test.go index 6b1069fc..0d7d879d 100644 --- a/src/runtime/race/testdata/mop_test.go +++ b/src/runtime/race/testdata/mop_test.go @@ -612,6 +612,8 @@ func TestNoRaceEnoughRegisters(t *testing.T) { } // emptyFunc should not be inlined. +// +//go:noinline func emptyFunc(x int) { if false { fmt.Println(x) @@ -1176,7 +1178,7 @@ func TestNoRaceHeapReallocation(t *testing.T) { // others. const n = 2 done := make(chan bool, n) - empty := func(p *int) {} + empty := func(p *int) { _ = p } for i := 0; i < n; i++ { ms := i go func() { @@ -1417,7 +1419,7 @@ func TestRaceInterCall2(t *testing.T) { func TestRaceFuncCall(t *testing.T) { c := make(chan bool, 1) - f := func(x, y int) {} + f := func(x, y int) { _ = y } x, y := 0, 0 go func() { y = 42 @@ -1804,6 +1806,7 @@ func TestRaceAsFunc2(t *testing.T) { x := 0 go func() { func(x int) { + _ = x }(x) c <- true }() @@ -1817,6 +1820,7 @@ func TestRaceAsFunc3(t *testing.T) { x := 0 go func() { func(x int) { + _ = x mu.Lock() }(x) // Read of x must be outside of the mutex. mu.Unlock() diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index c4a6d493..9c563892 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -43,7 +43,7 @@ // func runtime·raceread(addr uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·raceread(SB), NOSPLIT, $0-8 MOVQ AX, RARG1 MOVQ (SP), RARG2 @@ -69,7 +69,7 @@ TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 // func runtime·racewrite(addr uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·racewrite(SB), NOSPLIT, $0-8 MOVQ AX, RARG1 MOVQ (SP), RARG2 @@ -95,7 +95,7 @@ TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 // func runtime·racereadrange(addr, size uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 MOVQ AX, RARG1 MOVQ BX, RARG2 @@ -122,7 +122,7 @@ TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24 // func runtime·racewriterange(addr, size uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 MOVQ AX, RARG1 MOVQ BX, RARG2 diff --git a/src/runtime/rand.go b/src/runtime/rand.go index a66553fe..1739e9f8 100644 --- a/src/runtime/rand.go +++ b/src/runtime/rand.go @@ -7,9 +7,10 @@ package runtime import ( + "internal/byteorder" "internal/chacha8rand" "internal/goarch" - "runtime/internal/math" + "internal/runtime/math" "unsafe" _ "unsafe" // for go:linkname ) @@ -41,14 +42,15 @@ func randinit() { } seed := &globalRand.seed - if startupRand != nil { + if len(startupRand) >= 16 && + // Check that at least the first two words of startupRand weren't + // cleared by any libc initialization. + !allZero(startupRand[:8]) && !allZero(startupRand[8:16]) { for i, c := range startupRand { seed[i%len(seed)] ^= c } - clear(startupRand) - startupRand = nil } else { - if readRandom(seed[:]) != len(seed) { + if readRandom(seed[:]) != len(seed) || allZero(seed[:]) { // readRandom should never fail, but if it does we'd rather // not make Go binaries completely unusable, so make up // some random data based on the current time. @@ -58,6 +60,25 @@ func randinit() { } globalRand.state.Init(*seed) clear(seed[:]) + + if startupRand != nil { + // Overwrite startupRand instead of clearing it, in case cgo programs + // access it after we used it. + for len(startupRand) > 0 { + buf := make([]byte, 8) + for { + if x, ok := globalRand.state.Next(); ok { + byteorder.BEPutUint64(buf, x) + break + } + globalRand.state.Refill() + } + n := copy(startupRand, buf) + startupRand = startupRand[n:] + } + startupRand = nil + } + globalRand.init = true unlock(&globalRand.lock) } @@ -88,6 +109,14 @@ func readTimeRandom(r []byte) { } } +func allZero(b []byte) bool { + var acc byte + for _, x := range b { + acc |= x + } + return acc == 0 +} + // bootstrapRand returns a random uint64 from the global random generator. func bootstrapRand() uint64 { lock(&globalRand.lock) @@ -122,6 +151,8 @@ func rand32() uint32 { } // rand returns a random uint64 from the per-m chacha8 state. +// This is called from compiler-generated code. +// // Do not change signature: used via linkname from other packages. // //go:nosplit @@ -148,6 +179,11 @@ func rand() uint64 { } } +//go:linkname maps_rand internal/runtime/maps.rand +func maps_rand() uint64 { + return rand() +} + // mrandinit initializes the random state of an m. func mrandinit(mp *m) { var seed [4]uint64 diff --git a/src/runtime/rt0_js_wasm.s b/src/runtime/rt0_js_wasm.s index 34a60474..c7a0a263 100644 --- a/src/runtime/rt0_js_wasm.s +++ b/src/runtime/rt0_js_wasm.s @@ -53,12 +53,6 @@ TEXT wasm_export_getsp(SB),NOSPLIT,$0 Get SP Return -TEXT runtime·pause(SB), NOSPLIT, $0-8 - MOVD newsp+0(FP), SP - I32Const $1 - Set PAUSE - RETUNWIND - TEXT runtime·exit(SB), NOSPLIT, $0-4 I32Const $0 Call runtime·wasmExit(SB) diff --git a/src/runtime/rt0_wasip1_wasm.s b/src/runtime/rt0_wasip1_wasm.s index 6dc23930..a60566fe 100644 --- a/src/runtime/rt0_wasip1_wasm.s +++ b/src/runtime/rt0_wasip1_wasm.s @@ -14,3 +14,7 @@ TEXT _rt0_wasm_wasip1(SB),NOSPLIT,$0 Call wasm_pc_f_loop(SB) Return + +TEXT _rt0_wasm_wasip1_lib(SB),NOSPLIT,$0 + Call _rt0_wasm_wasip1(SB) + Return diff --git a/src/runtime/runtime-gdb.py b/src/runtime/runtime-gdb.py index 46f014fc..6d995151 100644 --- a/src/runtime/runtime-gdb.py +++ b/src/runtime/runtime-gdb.py @@ -160,7 +160,118 @@ class MapTypePrinter: return str(self.val.type) def children(self): - MapBucketCount = 8 # see internal/abi.go:MapBucketCount + fields = [f.name for f in self.val.type.strip_typedefs().target().fields()] + if 'buckets' in fields: + yield from self.old_map_children() + else: + yield from self.swiss_map_children() + + def swiss_map_children(self): + SwissMapGroupSlots = 8 # see internal/abi:SwissMapGroupSlots + + cnt = 0 + # Yield keys and elements in group. + # group is a value of type *group[K,V] + def group_slots(group): + ctrl = group['ctrl'] + + for i in xrange(SwissMapGroupSlots): + c = (ctrl >> (8*i)) & 0xff + if (c & 0x80) != 0: + # Empty or deleted + continue + + # Full + yield str(cnt), group['slots'][i]['key'] + yield str(cnt+1), group['slots'][i]['elem'] + + # The linker DWARF generation + # (cmd/link/internal/ld.(*dwctxt).synthesizemaptypesSwiss) records + # dirPtr as a **table[K,V], but it may actually be two different types: + # + # For "full size" maps (dirLen > 0), dirPtr is actually a pointer to + # variable length array *[dirLen]*table[K,V]. In other words, dirPtr + + # dirLen are a deconstructed slice []*table[K,V]. + # + # For "small" maps (dirLen <= 0), dirPtr is a pointer directly to a + # single group *group[K,V] containing the map slots. + # + # N.B. array() takes an _inclusive_ upper bound. + + # table[K,V] + table_type = self.val['dirPtr'].type.target().target() + + if self.val['dirLen'] <= 0: + # Small map + + # We need to find the group type we'll cast to. Since dirPtr isn't + # actually **table[K,V], we can't use the nice API of + # obj['field'].type, as that actually wants to dereference obj. + # Instead, search only via the type API. + ptr_group_type = None + for tf in table_type.fields(): + if tf.name != 'groups': + continue + groups_type = tf.type + for gf in groups_type.fields(): + if gf.name != 'data': + continue + # *group[K,V] + ptr_group_type = gf.type + + if ptr_group_type is None: + raise TypeError("unable to find table[K,V].groups.data") + + # group = (*group[K,V])(dirPtr) + group = self.val['dirPtr'].cast(ptr_group_type) + + yield from group_slots(group) + + return + + # Full size map. + + # *table[K,V] + ptr_table_type = table_type.pointer() + # [dirLen]*table[K,V] + array_ptr_table_type = ptr_table_type.array(self.val['dirLen']-1) + # *[dirLen]*table[K,V] + ptr_array_ptr_table_type = array_ptr_table_type.pointer() + # tables = (*[dirLen]*table[K,V])(dirPtr) + tables = self.val['dirPtr'].cast(ptr_array_ptr_table_type) + + cnt = 0 + for t in xrange(self.val['dirLen']): + table = tables[t] + table = table.dereference() + + groups = table['groups']['data'] + length = table['groups']['lengthMask'] + 1 + + # The linker DWARF generation + # (cmd/link/internal/ld.(*dwctxt).synthesizemaptypesSwiss) records + # groups.data as a *group[K,V], but it is actually a pointer to + # variable length array *[length]group[K,V]. + # + # N.B. array() takes an _inclusive_ upper bound. + + # group[K,V] + group_type = groups.type.target() + # [length]group[K,V] + array_group_type = group_type.array(length-1) + # *[length]group[K,V] + ptr_array_group_type = array_group_type.pointer() + # groups = (*[length]group[K,V])(groups.data) + groups = groups.cast(ptr_array_group_type) + groups = groups.dereference() + + for i in xrange(length): + group = groups[i] + yield from group_slots(group) + + + def old_map_children(self): + MapBucketCount = 8 # see internal/abi:OldMapBucketCount B = self.val['B'] buckets = self.val['buckets'] oldbuckets = self.val['oldbuckets'] @@ -385,7 +496,7 @@ goobjfile.pretty_printers.append(ifacematcher) class GoLenFunc(gdb.Function): "Length of strings, slices, maps or channels" - how = ((StringTypePrinter, 'len'), (SliceTypePrinter, 'len'), (MapTypePrinter, 'count'), (ChanTypePrinter, 'qcount')) + how = ((StringTypePrinter, 'len'), (SliceTypePrinter, 'len'), (MapTypePrinter, 'used'), (ChanTypePrinter, 'qcount')) def __init__(self): gdb.Function.__init__(self, "len") @@ -394,6 +505,12 @@ class GoLenFunc(gdb.Function): typename = str(obj.type) for klass, fld in self.how: if klass.pattern.match(typename) or paramtypematch(obj.type, klass.pattern): + if klass == MapTypePrinter: + fields = [f.name for f in self.val.type.strip_typedefs().target().fields()] + if 'buckets' in fields: + # Old maps. + fld = 'count' + return obj[fld] diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index 14561330..9c54d689 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "internal/abi" + "internal/goexperiment" "internal/testenv" "os" "os/exec" @@ -111,6 +112,44 @@ func checkCleanBacktrace(t *testing.T, backtrace string) { // TODO(mundaym): check for unknown frames (e.g. "??"). } +// checkPtraceScope checks the value of the kernel parameter ptrace_scope, +// skips the test when gdb cannot attach to the target process via ptrace. +// See issue 69932 +// +// 0 - Default attach security permissions. +// 1 - Restricted attach. Only child processes plus normal permissions. +// 2 - Admin-only attach. Only executables with CAP_SYS_PTRACE. +// 3 - No attach. No process may call ptrace at all. Irrevocable. +func checkPtraceScope(t *testing.T) { + if runtime.GOOS != "linux" { + return + } + + // If the Linux kernel does not have the YAMA module enabled, + // there will be no ptrace_scope file, which does not affect the tests. + path := "/proc/sys/kernel/yama/ptrace_scope" + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read file: %v", err) + } + value, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + t.Fatalf("failed converting value to int: %v", err) + } + switch value { + case 3: + t.Skip("skipping ptrace: Operation not permitted") + case 2: + if os.Geteuid() != 0 { + t.Skip("skipping ptrace: Operation not permitted with non-root user") + } + } +} + // NOTE: the maps below are allocated larger than abi.MapBucketCount // to ensure that they are not "optimized out". @@ -118,15 +157,19 @@ var helloSource = ` import "fmt" import "runtime" var gslice []string +// TODO(prattmic): Stack allocated maps initialized inline appear "optimized out" in GDB. +var smallmapvar map[string]string func main() { - mapvar := make(map[string]string, ` + strconv.FormatInt(abi.MapBucketCount+9, 10) + `) - slicemap := make(map[string][]string,` + strconv.FormatInt(abi.MapBucketCount+3, 10) + `) + smallmapvar = make(map[string]string) + mapvar := make(map[string]string, ` + strconv.FormatInt(abi.OldMapBucketCount+9, 10) + `) + slicemap := make(map[string][]string,` + strconv.FormatInt(abi.OldMapBucketCount+3, 10) + `) chanint := make(chan int, 10) chanstr := make(chan string, 10) chanint <- 99 chanint <- 11 chanstr <- "spongepants" chanstr <- "squarebob" + smallmapvar["abc"] = "def" mapvar["abc"] = "def" mapvar["ghi"] = "jkl" slicemap["a"] = []string{"b","c","d"} @@ -140,6 +183,7 @@ func main() { _ = ptrvar // set breakpoint here gslice = slicevar fmt.Printf("%v, %v, %v\n", slicemap, <-chanint, <-chanstr) + runtime.KeepAlive(smallmapvar) runtime.KeepAlive(mapvar) } // END_OF_PROGRAM ` @@ -193,6 +237,7 @@ func testGdbPython(t *testing.T, cgo bool) { t.Parallel() checkGdbVersion(t) checkGdbPython(t) + checkPtraceScope(t) dir := t.TempDir() @@ -254,6 +299,9 @@ func testGdbPython(t *testing.T, cgo bool) { "-ex", "echo BEGIN info goroutines\n", "-ex", "info goroutines", "-ex", "echo END\n", + "-ex", "echo BEGIN print smallmapvar\n", + "-ex", "print smallmapvar", + "-ex", "echo END\n", "-ex", "echo BEGIN print mapvar\n", "-ex", "print mapvar", "-ex", "echo END\n", @@ -306,6 +354,11 @@ func testGdbPython(t *testing.T, cgo bool) { t.Fatalf("info goroutines failed: %s", bl) } + printSmallMapvarRe := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def"}$`) + if bl := blocks["print smallmapvar"]; !printSmallMapvarRe.MatchString(bl) { + t.Fatalf("print smallmapvar failed: %s", bl) + } + printMapvarRe1 := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def", \[(0x[0-9a-f]+\s+)?"ghi"\] = (0x[0-9a-f]+\s+)?"jkl"}$`) printMapvarRe2 := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"ghi"\] = (0x[0-9a-f]+\s+)?"jkl", \[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def"}$`) if bl := blocks["print mapvar"]; !printMapvarRe1.MatchString(bl) && @@ -416,6 +469,7 @@ func TestGdbBacktrace(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) dir := t.TempDir() @@ -530,6 +584,7 @@ func TestGdbAutotmpTypes(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) if runtime.GOOS == "aix" && testing.Short() { t.Skip("TestGdbAutotmpTypes is too slow on aix/ppc64") @@ -576,10 +631,21 @@ func TestGdbAutotmpTypes(t *testing.T) { // Check that the backtrace matches the source code. types := []string{ "[]main.astruct", - "bucket", - "hash", "main.astruct", - "hash * map[string]main.astruct", + } + if goexperiment.SwissMap { + types = append(types, []string{ + "groupReference", + "table", + "map", + "map * map[string]main.astruct", + }...) + } else { + types = append(types, []string{ + "bucket", + "hash", + "hash * map[string]main.astruct", + }...) } for _, name := range types { if !strings.Contains(sgot, name) { @@ -604,6 +670,7 @@ func TestGdbConst(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) dir := t.TempDir() @@ -668,6 +735,7 @@ func TestGdbPanic(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) if runtime.GOOS == "windows" { t.Skip("no signals on windows") @@ -747,6 +815,7 @@ func TestGdbInfCallstack(t *testing.T) { t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) dir := t.TempDir() diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go index 03ef74b8..b47c5890 100644 --- a/src/runtime/runtime1.go +++ b/src/runtime/runtime1.go @@ -331,6 +331,7 @@ var debug struct { traceadvanceperiod int32 traceCheckStackOwnership int32 profstackdepth int32 + dataindependenttiming int32 // debug.malloc is used as a combined debug check // in the malloc function and should be set @@ -367,6 +368,7 @@ var dbgvars = []*dbgVar{ {name: "asynctimerchan", atomic: &debug.asynctimerchan}, {name: "cgocheck", value: &debug.cgocheck}, {name: "clobberfree", value: &debug.clobberfree}, + {name: "dataindependenttiming", value: &debug.dataindependenttiming}, {name: "disablethp", value: &debug.disablethp}, {name: "dontfreezetheworld", value: &debug.dontfreezetheworld}, {name: "efence", value: &debug.efence}, @@ -680,7 +682,6 @@ func reflect_resolveTypeOff(rtype unsafe.Pointer, off int32) unsafe.Pointer { // reflect_resolveTextOff is for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/cloudwego/frugal // - github.com/agiledragon/gomonkey/v2 // // Do not remove or change the type signature. @@ -725,3 +726,13 @@ func reflect_addReflectOff(ptr unsafe.Pointer) int32 { reflectOffsUnlock() return id } + +//go:linkname fips_getIndicator crypto/internal/fips140.getIndicator +func fips_getIndicator() uint8 { + return getg().fipsIndicator +} + +//go:linkname fips_setIndicator crypto/internal/fips140.setIndicator +func fips_setIndicator(indicator uint8) { + getg().fipsIndicator = indicator +} diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 4a1ee37a..e837c28a 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -8,8 +8,9 @@ import ( "internal/abi" "internal/chacha8rand" "internal/goarch" + "internal/goexperiment" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -170,33 +171,6 @@ type mutex struct { key uintptr } -// sleep and wakeup on one-time events. -// before any calls to notesleep or notewakeup, -// must call noteclear to initialize the Note. -// then, exactly one thread can call notesleep -// and exactly one thread can call notewakeup (once). -// once notewakeup has been called, the notesleep -// will return. future notesleep will return immediately. -// subsequent noteclear must be called only after -// previous notesleep has returned, e.g. it's disallowed -// to call noteclear straight after notewakeup. -// -// notetsleep is like notesleep but wakes up after -// a given number of nanoseconds even if the event -// has not yet happened. if a goroutine uses notetsleep to -// wake up early, it must wait to call noteclear until it -// can be sure that no other goroutine is calling -// notewakeup. -// -// notesleep/notetsleep are generally called on g0, -// notetsleepg is similar to notetsleep but is called on user g. -type note struct { - // Futex-based impl treats it as uint32 key, - // while sema-based impl as M* waitm. - // Used to be a union, but unions break precise GC. - key uintptr -} - type funcval struct { fn uintptr // variable-size, fn-specific data here @@ -493,6 +467,7 @@ type g struct { trackingStamp int64 // timestamp of when the G last started being tracked runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking lockedm muintptr + fipsIndicator uint8 sig uint32 writebuf []byte sigcode0 uintptr @@ -514,7 +489,8 @@ type g struct { // current in-progress goroutine profile goroutineProfiled goroutineProfileStateHolder - coroarg *coro // argument during coroutine transfers + coroarg *coro // argument during coroutine transfers + syncGroup *synctestGroup // Per-G tracer state. trace gTraceState @@ -597,7 +573,7 @@ type m struct { createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it. lockedExt uint32 // tracking for external LockOSThread lockedInt uint32 // tracking for internal lockOSThread - nextwaitm muintptr // next m waiting for lock + mWaitList mWaitList // list of runtime lock waiters mLockProfile mLockProfile // fields relating to runtime.lock contention profStack []uintptr // used for memory/block/mutex stack traces @@ -645,6 +621,12 @@ type m struct { // Up to 10 locks held by this m, maintained by the lock ranking code. locksHeldLen int locksHeld [10]heldLockInfo + + // Size the runtime.m structure so it fits in the 2048-byte size class, and + // not in the next-smallest (1792-byte) size class. That leaves the 11 low + // bits of muintptr values available for flags, as required for + // GOEXPERIMENT=spinbitmutex. + _ [goexperiment.SpinbitMutexInt * 700 * (2 - goarch.PtrSize/4)]byte } type p struct { @@ -1083,6 +1065,7 @@ const ( waitReasonSyncMutexLock // "sync.Mutex.Lock" waitReasonSyncRWMutexRLock // "sync.RWMutex.RLock" waitReasonSyncRWMutexLock // "sync.RWMutex.Lock" + waitReasonSyncWaitGroupWait // "sync.WaitGroup.Wait" waitReasonTraceReaderBlocked // "trace reader (blocked)" waitReasonWaitForGCCycle // "wait for GC cycle" waitReasonGCWorkerIdle // "GC worker (idle)" @@ -1097,6 +1080,11 @@ const ( waitReasonPageTraceFlush // "page trace flush" waitReasonCoroutine // "coroutine" waitReasonGCWeakToStrongWait // "GC weak to strong wait" + waitReasonSynctestRun // "synctest.Run" + waitReasonSynctestWait // "synctest.Wait" + waitReasonSynctestChanReceive // "chan receive (synctest)" + waitReasonSynctestChanSend // "chan send (synctest)" + waitReasonSynctestSelect // "select (synctest)" ) var waitReasonStrings = [...]string{ @@ -1124,6 +1112,7 @@ var waitReasonStrings = [...]string{ waitReasonSyncMutexLock: "sync.Mutex.Lock", waitReasonSyncRWMutexRLock: "sync.RWMutex.RLock", waitReasonSyncRWMutexLock: "sync.RWMutex.Lock", + waitReasonSyncWaitGroupWait: "sync.WaitGroup.Wait", waitReasonTraceReaderBlocked: "trace reader (blocked)", waitReasonWaitForGCCycle: "wait for GC cycle", waitReasonGCWorkerIdle: "GC worker (idle)", @@ -1138,6 +1127,11 @@ var waitReasonStrings = [...]string{ waitReasonPageTraceFlush: "page trace flush", waitReasonCoroutine: "coroutine", waitReasonGCWeakToStrongWait: "GC weak to strong wait", + waitReasonSynctestRun: "synctest.Run", + waitReasonSynctestWait: "synctest.Wait", + waitReasonSynctestChanReceive: "chan receive (synctest)", + waitReasonSynctestChanSend: "chan send (synctest)", + waitReasonSynctestSelect: "select (synctest)", } func (w waitReason) String() string { @@ -1176,6 +1170,26 @@ var isWaitingForGC = [len(waitReasonStrings)]bool{ waitReasonFlushProcCaches: true, } +func (w waitReason) isIdleInSynctest() bool { + return isIdleInSynctest[w] +} + +// isIdleInSynctest indicates that a goroutine is considered idle by synctest.Wait. +var isIdleInSynctest = [len(waitReasonStrings)]bool{ + waitReasonChanReceiveNilChan: true, + waitReasonChanSendNilChan: true, + waitReasonSelectNoCases: true, + waitReasonSleep: true, + waitReasonSyncCondWait: true, + waitReasonSyncWaitGroupWait: true, + waitReasonCoroutine: true, + waitReasonSynctestRun: true, + waitReasonSynctestWait: true, + waitReasonSynctestChanReceive: true, + waitReasonSynctestChanSend: true, + waitReasonSynctestSelect: true, +} + var ( allm *m gomaxprocs int32 diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go index c1bf7f87..f23581ac 100644 --- a/src/runtime/runtime_test.go +++ b/src/runtime/runtime_test.go @@ -7,7 +7,10 @@ package runtime_test import ( "flag" "fmt" + "internal/cpu" + "internal/runtime/atomic" "io" + "math/bits" . "runtime" "runtime/debug" "slices" @@ -539,3 +542,280 @@ func TestTimediv(t *testing.T) { }) } } + +func BenchmarkProcYield(b *testing.B) { + benchN := func(n uint32) func(*testing.B) { + return func(b *testing.B) { + for i := 0; i < b.N; i++ { + ProcYield(n) + } + } + } + + b.Run("1", benchN(1)) + b.Run("10", benchN(10)) + b.Run("30", benchN(30)) // active_spin_cnt in lock_sema.go and lock_futex.go + b.Run("100", benchN(100)) + b.Run("1000", benchN(1000)) +} + +func BenchmarkOSYield(b *testing.B) { + for i := 0; i < b.N; i++ { + OSYield() + } +} + +func BenchmarkMutexContention(b *testing.B) { + // Measure throughput of a single mutex with all threads contending + // + // Share a single counter across all threads. Progress from any thread is + // progress for the benchmark as a whole. We don't measure or give points + // for fairness here, arbitrary delay to any given thread's progress is + // invisible and allowed. + // + // The cache line that holds the count value will need to move between + // processors, but not as often as the cache line that holds the mutex. The + // mutex protects access to the count value, which limits contention on that + // cache line. This is a simple design, but it helps to make the behavior of + // the benchmark clear. Most real uses of mutex will protect some number of + // cache lines anyway. + + var state struct { + _ cpu.CacheLinePad + lock Mutex + _ cpu.CacheLinePad + count atomic.Int64 + _ cpu.CacheLinePad + } + + procs := GOMAXPROCS(0) + var wg sync.WaitGroup + for range procs { + wg.Add(1) + go func() { + defer wg.Done() + for { + Lock(&state.lock) + ours := state.count.Add(1) + Unlock(&state.lock) + if ours >= int64(b.N) { + return + } + } + }() + } + wg.Wait() +} + +func BenchmarkMutexCapture(b *testing.B) { + + // Measure mutex fairness. + // + // Have several threads contend for a single mutex value. Measure how + // effectively a single thread is able to capture the lock and report the + // duration of those "streak" events. Measure how long other individual + // threads need to wait between their turns with the lock. Report the + // duration of those "starve" events. + // + // Report in terms of wall clock time (assuming a constant time per + // lock/unlock pair) rather than number of locks/unlocks. This keeps + // timekeeping overhead out of the critical path, and avoids giving an + // advantage to lock/unlock implementations that take less time per + // operation. + + var state struct { + _ cpu.CacheLinePad + lock Mutex + _ cpu.CacheLinePad + count atomic.Int64 + _ cpu.CacheLinePad + } + + procs := GOMAXPROCS(0) + var wg sync.WaitGroup + histograms := make(chan [2][65]int) + for range procs { + wg.Add(1) + go func() { + var ( + prev int64 + streak int64 + histogram [2][65]int + ) + for { + Lock(&state.lock) + ours := state.count.Add(1) + Unlock(&state.lock) + delta := ours - prev - 1 + prev = ours + if delta == 0 { + streak++ + } else { + histogram[0][bits.LeadingZeros64(uint64(streak))]++ + histogram[1][bits.LeadingZeros64(uint64(delta))]++ + streak = 1 + } + if ours >= int64(b.N) { + wg.Done() + if delta == 0 { + histogram[0][bits.LeadingZeros64(uint64(streak))]++ + histogram[1][bits.LeadingZeros64(uint64(delta))]++ + } + histograms <- histogram + return + } + } + }() + } + + wg.Wait() + b.StopTimer() + + var histogram [2][65]int + for range procs { + h := <-histograms + for i := range h { + for j := range h[i] { + histogram[i][j] += h[i][j] + } + } + } + + percentile := func(h [65]int, p float64) int { + sum := 0 + for i, v := range h { + bound := uint64(1<<63) >> i + sum += int(bound) * v + } + + // Imagine that the longest streak / starvation events were instead half + // as long but twice in number. (Note that we've pre-multiplied by the + // [lower] "bound" value.) Continue those splits until we meet the + // percentile target. + part := 0 + for i, v := range h { + bound := uint64(1<<63) >> i + part += int(bound) * v + // have we trimmed off enough at the head to dip below the percentile goal + if float64(sum-part) < float64(sum)*p { + return int(bound) + } + } + + return 0 + } + + perOp := float64(b.Elapsed().Nanoseconds()) / float64(b.N) + b.ReportMetric(perOp*float64(percentile(histogram[0], 1.0)), "ns/streak-p100") + b.ReportMetric(perOp*float64(percentile(histogram[0], 0.9)), "ns/streak-p90") + b.ReportMetric(perOp*float64(percentile(histogram[1], 1.0)), "ns/starve-p100") + b.ReportMetric(perOp*float64(percentile(histogram[1], 0.9)), "ns/starve-p90") +} + +func BenchmarkMutexHandoff(b *testing.B) { + testcase := func(delay func(l *Mutex)) func(b *testing.B) { + return func(b *testing.B) { + if workers := 2; GOMAXPROCS(0) < workers { + b.Skipf("requires GOMAXPROCS >= %d", workers) + } + + // Measure latency of mutex handoff between threads. + // + // Hand off a runtime.mutex between two threads, one running a + // "coordinator" goroutine and the other running a "worker" + // goroutine. We don't override the runtime's typical + // goroutine/thread mapping behavior. + // + // Measure the latency, starting when the coordinator enters a call + // to runtime.unlock and ending when the worker's call to + // runtime.lock returns. The benchmark can specify a "delay" + // function to simulate the length of the mutex-holder's critical + // section, including to arrange for the worker's thread to be in + // either the "spinning" or "sleeping" portions of the runtime.lock2 + // implementation. Measurement starts after any such "delay". + // + // The two threads' goroutines communicate their current position to + // each other in a non-blocking way via the "turn" state. + + var state struct { + _ cpu.CacheLinePad + lock Mutex + _ cpu.CacheLinePad + turn atomic.Int64 + _ cpu.CacheLinePad + } + + var delta atomic.Int64 + var wg sync.WaitGroup + + // coordinator: + // - acquire the mutex + // - set the turn to 2 mod 4, instructing the worker to begin its Lock call + // - wait until the mutex is contended + // - wait a bit more so the worker can commit to its sleep + // - release the mutex and wait for it to be our turn (0 mod 4) again + wg.Add(1) + go func() { + defer wg.Done() + var t int64 + for range b.N { + Lock(&state.lock) + state.turn.Add(2) + delay(&state.lock) + t -= Nanotime() // start the timer + Unlock(&state.lock) + for state.turn.Load()&0x2 != 0 { + } + } + state.turn.Add(1) + delta.Add(t) + }() + + // worker: + // - wait until its our turn (2 mod 4) + // - acquire and release the mutex + // - switch the turn counter back to the coordinator (0 mod 4) + wg.Add(1) + go func() { + defer wg.Done() + var t int64 + for { + switch state.turn.Load() & 0x3 { + case 0: + case 1, 3: + delta.Add(t) + return + case 2: + Lock(&state.lock) + t += Nanotime() // stop the timer + Unlock(&state.lock) + state.turn.Add(2) + } + } + }() + + wg.Wait() + b.ReportMetric(float64(delta.Load())/float64(b.N), "ns/op") + } + } + + b.Run("Solo", func(b *testing.B) { + var lock Mutex + for range b.N { + Lock(&lock) + Unlock(&lock) + } + }) + + b.Run("FastPingPong", testcase(func(l *Mutex) {})) + b.Run("SlowPingPong", testcase(func(l *Mutex) { + // Wait for the worker to stop spinning and prepare to sleep + for !MutexContended(l) { + } + // Wait a bit longer so the OS can finish committing the worker to its + // sleep. Balance consistency against getting enough iterations. + const extraNs = 10e3 + for t0 := Nanotime(); Nanotime()-t0 < extraNs; { + } + })) +} diff --git a/src/runtime/select.go b/src/runtime/select.go index 17c49d74..0b1d1449 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "internal/runtime/sys" "unsafe" ) @@ -27,7 +28,7 @@ var ( ) func selectsetpc(pc *uintptr) { - *pc = getcallerpc() + *pc = sys.GetCallerPC() } func sellock(scases []scase, lockorder []uint16) { @@ -119,6 +120,7 @@ func block() { // Also, if the chosen scase was a receive operation, it reports whether // a value was received. func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, block bool) (int, bool) { + gp := getg() if debugSelect { print("select: cas0=", cas0, "\n") } @@ -164,6 +166,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo // generate permuted order norder := 0 + allSynctest := true for i := range scases { cas := &scases[i] @@ -173,6 +176,14 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo continue } + if cas.c.synctest { + if getg().syncGroup == nil { + panic(plainError("select on synctest channel from outside bubble")) + } + } else { + allSynctest = false + } + if cas.c.timer != nil { cas.c.timer.maybeRunChan() } @@ -185,6 +196,13 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo pollorder = pollorder[:norder] lockorder = lockorder[:norder] + waitReason := waitReasonSelect + if gp.syncGroup != nil && allSynctest { + // Every channel selected on is in a synctest bubble, + // so this goroutine will count as idle while selecting. + waitReason = waitReasonSynctestSelect + } + // sort the cases by Hchan address to get the locking order. // simple heap sort, to guarantee n log n time and constant stack footprint. for i := range lockorder { @@ -234,7 +252,6 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo sellock(scases, lockorder) var ( - gp *g sg *sudog c *hchan k *scase @@ -290,7 +307,6 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo } // pass 2 - enqueue on all chans - gp = getg() if gp.waiting != nil { throw("gp.waiting != nil") } @@ -332,7 +348,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo // changes and when we set gp.activeStackChans is not safe for // stack shrinking. gp.parkingOnChan.Store(true) - gopark(selparkcommit, nil, waitReasonSelect, traceBlockSelect, 1) + gopark(selparkcommit, nil, waitReason, traceBlockSelect, 1) gp.activeStackChans = false sellock(scases, lockorder) diff --git a/src/runtime/sema.go b/src/runtime/sema.go index f6b1b84f..18ada5a6 100644 --- a/src/runtime/sema.go +++ b/src/runtime/sema.go @@ -90,8 +90,8 @@ func sync_runtime_Semrelease(addr *uint32, handoff bool, skipframes int) { semrelease1(addr, handoff, skipframes) } -//go:linkname sync_runtime_SemacquireMutex sync.runtime_SemacquireMutex -func sync_runtime_SemacquireMutex(addr *uint32, lifo bool, skipframes int) { +//go:linkname internal_sync_runtime_SemacquireMutex internal/sync.runtime_SemacquireMutex +func internal_sync_runtime_SemacquireMutex(addr *uint32, lifo bool, skipframes int) { semacquire1(addr, lifo, semaBlockProfile|semaMutexProfile, skipframes, waitReasonSyncMutexLock) } @@ -105,11 +105,21 @@ func sync_runtime_SemacquireRWMutex(addr *uint32, lifo bool, skipframes int) { semacquire1(addr, lifo, semaBlockProfile|semaMutexProfile, skipframes, waitReasonSyncRWMutexLock) } +//go:linkname sync_runtime_SemacquireWaitGroup sync.runtime_SemacquireWaitGroup +func sync_runtime_SemacquireWaitGroup(addr *uint32) { + semacquire1(addr, false, semaBlockProfile, 0, waitReasonSyncWaitGroupWait) +} + //go:linkname poll_runtime_Semrelease internal/poll.runtime_Semrelease func poll_runtime_Semrelease(addr *uint32) { semrelease(addr) } +//go:linkname internal_sync_runtime_Semrelease internal/sync.runtime_Semrelease +func internal_sync_runtime_Semrelease(addr *uint32, handoff bool, skipframes int) { + semrelease1(addr, handoff, skipframes) +} + func readyWithTime(s *sudog, traceskip int) { if s.releasetime != 0 { s.releasetime = cputicks() @@ -619,6 +629,10 @@ func notifyListNotifyAll(l *notifyList) { for s != nil { next := s.next s.next = nil + if s.g.syncGroup != nil && getg().syncGroup != s.g.syncGroup { + println("semaphore wake of synctest goroutine", s.g.goid, "from outside bubble") + panic("semaphore wake of synctest goroutine from outside bubble") + } readyWithTime(s, 4) s = next } @@ -672,6 +686,10 @@ func notifyListNotifyOne(l *notifyList) { } unlock(&l.lock) s.next = nil + if s.g.syncGroup != nil && getg().syncGroup != s.g.syncGroup { + println("semaphore wake of synctest goroutine", s.g.goid, "from outside bubble") + panic("semaphore wake of synctest goroutine from outside bubble") + } readyWithTime(s, 4) return } @@ -687,7 +705,7 @@ func notifyListCheck(sz uintptr) { } } -//go:linkname sync_nanotime sync.runtime_nanotime -func sync_nanotime() int64 { +//go:linkname internal_sync_nanotime internal/sync.runtime_nanotime +func internal_sync_nanotime() int64 { return nanotime() } diff --git a/src/runtime/signal_arm64.go b/src/runtime/signal_arm64.go index 4a96b3c2..af7d29f9 100644 --- a/src/runtime/signal_arm64.go +++ b/src/runtime/signal_arm64.go @@ -9,7 +9,7 @@ package runtime import ( "internal/abi" "internal/goarch" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/signal_linux_s390x.go b/src/runtime/signal_linux_s390x.go index 18c3b115..54e9d1fb 100644 --- a/src/runtime/signal_linux_s390x.go +++ b/src/runtime/signal_linux_s390x.go @@ -7,7 +7,7 @@ package runtime import ( "internal/abi" "internal/goarch" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/signal_loong64.go b/src/runtime/signal_loong64.go index ac842c0c..970af01c 100644 --- a/src/runtime/signal_loong64.go +++ b/src/runtime/signal_loong64.go @@ -53,9 +53,10 @@ func dumpregs(c *sigctxt) { //go:nowritebarrierrec func (c *sigctxt) sigpc() uintptr { return uintptr(c.pc()) } -func (c *sigctxt) sigsp() uintptr { return uintptr(c.sp()) } -func (c *sigctxt) siglr() uintptr { return uintptr(c.link()) } -func (c *sigctxt) fault() uintptr { return uintptr(c.sigaddr()) } +func (c *sigctxt) setsigpc(x uint64) { c.set_pc(x) } +func (c *sigctxt) sigsp() uintptr { return uintptr(c.sp()) } +func (c *sigctxt) siglr() uintptr { return uintptr(c.link()) } +func (c *sigctxt) fault() uintptr { return uintptr(c.sigaddr()) } // preparePanic sets up the stack to look like a call to sigpanic. func (c *sigctxt) preparePanic(sig uint32, gp *g) { diff --git a/src/runtime/signal_mipsx.go b/src/runtime/signal_mipsx.go index ba926551..924e654c 100644 --- a/src/runtime/signal_mipsx.go +++ b/src/runtime/signal_mipsx.go @@ -8,7 +8,7 @@ package runtime import ( "internal/abi" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/signal_ppc64x.go b/src/runtime/signal_ppc64x.go index b5722f99..20f874c2 100644 --- a/src/runtime/signal_ppc64x.go +++ b/src/runtime/signal_ppc64x.go @@ -8,7 +8,7 @@ package runtime import ( "internal/abi" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index 6f40f440..96628d6b 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -9,7 +9,7 @@ package runtime import ( "internal/abi" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -405,7 +405,7 @@ func sigFetchG(c *sigctxt) *g { // bottom of the signal stack. Fetch from there. // TODO: in efence mode, stack is sysAlloc'd, so this wouldn't // work. - sp := getcallersp() + sp := sys.GetCallerSP() s := spanOf(sp) if s != nil && s.state.get() == mSpanManual && s.base() < sp && sp < s.limit { gp := *(**g)(unsafe.Pointer(s.base())) @@ -479,7 +479,7 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) { var gsignalStack gsignalStack setStack := adjustSignalStack(sig, gp.m, &gsignalStack) if setStack { - gp.m.gsignal.stktopsp = getcallersp() + gp.m.gsignal.stktopsp = sys.GetCallerSP() } if gp.stackguard0 == stackFork { @@ -584,15 +584,23 @@ func adjustSignalStack(sig uint32, mp *m, gsigStack *gsignalStack) bool { } // sp is not within gsignal stack, g0 stack, or sigaltstack. Bad. + // Call indirectly to avoid nosplit stack overflow on OpenBSD. + adjustSignalStack2Indirect(sig, sp, mp, st.ss_flags&_SS_DISABLE != 0) + return false +} + +var adjustSignalStack2Indirect = adjustSignalStack2 + +//go:nosplit +func adjustSignalStack2(sig uint32, sp uintptr, mp *m, ssDisable bool) { setg(nil) needm(true) - if st.ss_flags&_SS_DISABLE != 0 { + if ssDisable { noSignalStack(sig) } else { sigNotOnStack(sig, sp, mp) } dropm() - return false } // crashing is the number of m's we have waited for when implementing diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index 4b7960c1..b0c653ee 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -6,7 +6,7 @@ package runtime import ( "internal/abi" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/sizeof_test.go b/src/runtime/sizeof_test.go index 43aba98d..a5dc8aed 100644 --- a/src/runtime/sizeof_test.go +++ b/src/runtime/sizeof_test.go @@ -20,7 +20,7 @@ func TestSizeof(t *testing.T) { _32bit uintptr // size on 32bit platforms _64bit uintptr // size on 64bit platforms }{ - {runtime.G{}, 272, 432}, // g, but exported for testing + {runtime.G{}, 280, 440}, // g, but exported for testing {runtime.Sudog{}, 56, 88}, // sudog, but exported for testing } diff --git a/src/runtime/slice.go b/src/runtime/slice.go index 78475735..79d3f6c0 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -7,8 +7,8 @@ package runtime import ( "internal/abi" "internal/goarch" - "runtime/internal/math" - "runtime/internal/sys" + "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) @@ -18,7 +18,7 @@ type slice struct { cap int } -// A notInHeapSlice is a slice backed by runtime/internal/sys.NotInHeap memory. +// A notInHeapSlice is a slice backed by internal/runtime/sys.NotInHeap memory. type notInHeapSlice struct { array *notInHeap len int @@ -73,7 +73,7 @@ func makeslicecopy(et *_type, tolen int, fromlen int, from unsafe.Pointer) unsaf } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(makeslicecopy) racereadrangepc(from, copymem, callerpc, pc) } @@ -177,7 +177,7 @@ func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer { func growslice(oldPtr unsafe.Pointer, newLen, oldCap, num int, et *_type) slice { oldLen := newLen - num if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadrangepc(oldPtr, uintptr(oldLen*int(et.Size_)), callerpc, abi.FuncPCABIInternal(growslice)) } if msanenabled { @@ -368,7 +368,7 @@ func slicecopy(toPtr unsafe.Pointer, toLen int, fromPtr unsafe.Pointer, fromLen size := uintptr(n) * width if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(slicecopy) racereadrangepc(fromPtr, size, callerpc, pc) racewriterangepc(toPtr, size, callerpc, pc) diff --git a/src/runtime/stack.go b/src/runtime/stack.go index d43c6ace..8f11f54c 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -10,7 +10,7 @@ import ( "internal/goarch" "internal/goos" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -416,7 +416,7 @@ func stackalloc(n uint32) stack { } if traceAllocFreeEnabled() { - trace := traceTryAcquire() + trace := traceAcquire() if trace.ok() { trace.GoroutineStackAlloc(uintptr(v), uintptr(n)) traceRelease(trace) @@ -466,7 +466,7 @@ func stackfree(stk stack) { return } if traceAllocFreeEnabled() { - trace := traceTryAcquire() + trace := traceAcquire() if trace.ok() { trace.GoroutineStackFree(uintptr(v)) traceRelease(trace) @@ -722,22 +722,12 @@ func adjustframe(frame *stkframe, adjinfo *adjustinfo) { // we call into morestack.) continue } - ptrdata := obj.ptrdata() - gcdata := obj.gcdata() - var s *mspan - if obj.useGCProg() { - // See comments in mgcmark.go:scanstack - s = materializeGCProg(ptrdata, gcdata) - gcdata = (*byte)(unsafe.Pointer(s.startAddr)) - } - for i := uintptr(0); i < ptrdata; i += goarch.PtrSize { - if *addb(gcdata, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 { + ptrBytes, gcData := obj.gcdata() + for i := uintptr(0); i < ptrBytes; i += goarch.PtrSize { + if *addb(gcData, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 { adjustpointer(adjinfo, unsafe.Pointer(p+i)) } } - if s != nil { - dematerializeGCProg(s) - } } } } @@ -1288,24 +1278,14 @@ type stackObjectRecord struct { // if non-negative, offset from argp off int32 size int32 - _ptrdata int32 // ptrdata, or -ptrdata is GC prog is used + ptrBytes int32 gcdataoff uint32 // offset to gcdata from moduledata.rodata } -func (r *stackObjectRecord) useGCProg() bool { - return r._ptrdata < 0 -} - -func (r *stackObjectRecord) ptrdata() uintptr { - x := r._ptrdata - if x < 0 { - return uintptr(-x) - } - return uintptr(x) -} - -// gcdata returns pointer map or GC prog of the type. -func (r *stackObjectRecord) gcdata() *byte { +// gcdata returns the number of bytes that contain pointers, and +// a ptr/nonptr bitmask covering those bytes. +// Note that this bitmask might be larger than internal/abi.MaxPtrmaskBytes. +func (r *stackObjectRecord) gcdata() (uintptr, *byte) { ptr := uintptr(unsafe.Pointer(r)) var mod *moduledata for datap := &firstmoduledata; datap != nil; datap = datap.next { @@ -1318,7 +1298,7 @@ func (r *stackObjectRecord) gcdata() *byte { // you may have made a copy of a stackObjectRecord. // You must use the original pointer. res := mod.rodata + uintptr(r.gcdataoff) - return (*byte)(unsafe.Pointer(res)) + return uintptr(r.ptrBytes), (*byte)(unsafe.Pointer(res)) } // This is exported as ABI0 via linkname so obj can call it. diff --git a/src/runtime/stkframe.go b/src/runtime/stkframe.go index 42b69477..819b7f6c 100644 --- a/src/runtime/stkframe.go +++ b/src/runtime/stkframe.go @@ -7,7 +7,7 @@ package runtime import ( "internal/abi" "internal/goarch" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -264,9 +264,6 @@ var methodValueCallFrameObjs [1]stackObjectRecord // initialized in stackobjecti func stkobjinit() { var abiRegArgsEface any = abi.RegArgs{} abiRegArgsType := efaceOf(&abiRegArgsEface)._type - if abiRegArgsType.Kind_&abi.KindGCProg != 0 { - throw("abiRegArgsType needs GC Prog, update methodValueCallFrameObjs") - } // Set methodValueCallFrameObjs[0].gcdataoff so that // stackObjectRecord.gcdata() will work correctly with it. ptr := uintptr(unsafe.Pointer(&methodValueCallFrameObjs[0])) @@ -283,7 +280,7 @@ func stkobjinit() { methodValueCallFrameObjs[0] = stackObjectRecord{ off: -int32(alignUp(abiRegArgsType.Size_, 8)), // It's always the highest address local. size: int32(abiRegArgsType.Size_), - _ptrdata: int32(abiRegArgsType.PtrBytes), - gcdataoff: uint32(uintptr(unsafe.Pointer(abiRegArgsType.GCData)) - mod.rodata), + ptrBytes: int32(abiRegArgsType.PtrBytes), + gcdataoff: uint32(uintptr(unsafe.Pointer(getGCMask(abiRegArgsType))) - mod.rodata), } } diff --git a/src/runtime/string.go b/src/runtime/string.go index 5bdb25b9..e43f4cca 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -8,6 +8,7 @@ import ( "internal/abi" "internal/bytealg" "internal/goarch" + "internal/runtime/sys" "unsafe" ) @@ -50,8 +51,8 @@ func concatstrings(buf *tmpBuf, a []string) string { } s, b := rawstringtmp(buf, l) for _, x := range a { - copy(b, x) - b = b[len(x):] + n := copy(b, x) + b = b[n:] } return s } @@ -72,22 +73,55 @@ func concatstring5(buf *tmpBuf, a0, a1, a2, a3, a4 string) string { return concatstrings(buf, []string{a0, a1, a2, a3, a4}) } +// concatbytes implements a Go string concatenation x+y+z+... returning a slice +// of bytes. +// The operands are passed in the slice a. +func concatbytes(a []string) []byte { + l := 0 + for _, x := range a { + n := len(x) + if l+n < l { + throw("string concatenation too long") + } + l += n + } + if l == 0 { + // This is to match the return type of the non-optimized concatenation. + return []byte{} + } + + b := rawbyteslice(l) + offset := 0 + for _, x := range a { + copy(b[offset:], x) + offset += len(x) + } + + return b +} + +func concatbyte2(a0, a1 string) []byte { + return concatbytes([]string{a0, a1}) +} + +func concatbyte3(a0, a1, a2 string) []byte { + return concatbytes([]string{a0, a1, a2}) +} + +func concatbyte4(a0, a1, a2, a3 string) []byte { + return concatbytes([]string{a0, a1, a2, a3}) +} + +func concatbyte5(a0, a1, a2, a3, a4 string) []byte { + return concatbytes([]string{a0, a1, a2, a3, a4}) +} + // slicebytetostring converts a byte slice to a string. // It is inserted by the compiler into generated code. // ptr is a pointer to the first element of the slice; // n is the length of the slice. // Buf is a fixed-size buffer for the result, // it is not nil if the result does not escape. -// -// slicebytetostring should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname slicebytetostring func slicebytetostring(buf *tmpBuf, ptr *byte, n int) string { if n == 0 { // Turns out to be a relatively common case. @@ -98,7 +132,7 @@ func slicebytetostring(buf *tmpBuf, ptr *byte, n int) string { if raceenabled { racereadrangepc(unsafe.Pointer(ptr), uintptr(n), - getcallerpc(), + sys.GetCallerPC(), abi.FuncPCABIInternal(slicebytetostring)) } if msanenabled { @@ -161,7 +195,7 @@ func slicebytetostringtmp(ptr *byte, n int) string { if raceenabled && n > 0 { racereadrangepc(unsafe.Pointer(ptr), uintptr(n), - getcallerpc(), + sys.GetCallerPC(), abi.FuncPCABIInternal(slicebytetostringtmp)) } if msanenabled && n > 0 { @@ -213,7 +247,7 @@ func slicerunetostring(buf *tmpBuf, a []rune) string { if raceenabled && len(a) > 0 { racereadrangepc(unsafe.Pointer(&a[0]), uintptr(len(a))*unsafe.Sizeof(a[0]), - getcallerpc(), + sys.GetCallerPC(), abi.FuncPCABIInternal(slicerunetostring)) } if msanenabled && len(a) > 0 { diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 2aeb4774..55153a20 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -97,7 +97,6 @@ func badsystemstack() { // Notable members of the hall of shame include: // - github.com/bytedance/sonic // - github.com/chenzhuoyu/iasm -// - github.com/cloudwego/frugal // - github.com/dgraph-io/ristretto // - github.com/outcaste-io/ristretto // @@ -132,7 +131,6 @@ func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { // Notable members of the hall of shame include: // - github.com/bytedance/sonic // - github.com/cloudwego/dynamicgo -// - github.com/cloudwego/frugal // - github.com/ebitengine/purego // - github.com/tetratelabs/wazero // - github.com/ugorji/go/codec @@ -270,7 +268,7 @@ func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSi // Notable members of the hall of shame include: // - github.com/sagernet/sing-tun // - github.com/slackhq/nebula -// - github.com/tailscale/wireguard-go +// - golang.zx2c4.com/wireguard // // Do not remove or change the type signature. // See go.dev/issue/67401. @@ -309,71 +307,11 @@ func goexit(neverCallThisFunction) // data dependency ordering. func publicationBarrier() -// getcallerpc returns the program counter (PC) of its caller's caller. -// getcallersp returns the stack pointer (SP) of its caller's caller. -// The implementation may be a compiler intrinsic; there is not -// necessarily code implementing this on every platform. -// -// For example: -// -// func f(arg1, arg2, arg3 int) { -// pc := getcallerpc() -// sp := getcallersp() -// } -// -// These two lines find the PC and SP immediately following -// the call to f (where f will return). -// -// The call to getcallerpc and getcallersp must be done in the -// frame being asked about. -// -// The result of getcallersp is correct at the time of the return, -// but it may be invalidated by any subsequent call to a function -// that might relocate the stack in order to grow or shrink it. -// A general rule is that the result of getcallersp should be used -// immediately and can only be passed to nosplit functions. - -//go:noescape -func getcallerpc() uintptr - -//go:noescape -func getcallersp() uintptr // implemented as an intrinsic on all platforms - -// getclosureptr returns the pointer to the current closure. -// getclosureptr can only be used in an assignment statement -// at the entry of a function. Moreover, go:nosplit directive -// must be specified at the declaration of caller function, -// so that the function prolog does not clobber the closure register. -// for example: -// -// //go:nosplit -// func f(arg1, arg2, arg3 int) { -// dx := getclosureptr() -// } -// -// The compiler rewrites calls to this function into instructions that fetch the -// pointer from a well-known register (DX on x86 architecture, etc.) directly. -// -// WARNING: PGO-based devirtualization cannot detect that caller of -// getclosureptr require closure context, and thus must maintain a list of -// these functions, which is in -// cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. -func getclosureptr() uintptr - //go:noescape func asmcgocall(fn, arg unsafe.Pointer) int32 func morestack() -// morestack_noctxt should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname morestack_noctxt func morestack_noctxt() func rt0_go() @@ -433,6 +371,8 @@ func alignDown(n, a uintptr) uintptr { } // divRoundUp returns ceil(n / a). +// +//go:nosplit func divRoundUp(n, a uintptr) uintptr { // a is generally a power of two. This will get inlined and // the compiler will optimize the division. @@ -465,7 +405,6 @@ func gcWriteBarrier1() // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // // Do not remove or change the type signature. // See go.dev/issue/67401. diff --git a/src/runtime/stubs_nonwasm.go b/src/runtime/stubs_nonwasm.go new file mode 100644 index 00000000..fa4058bc --- /dev/null +++ b/src/runtime/stubs_nonwasm.go @@ -0,0 +1,10 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !wasm + +package runtime + +// pause is only used on wasm. +func pause(newsp uintptr) { panic("unreachable") } diff --git a/src/runtime/stubs_wasm.go b/src/runtime/stubs_wasm.go new file mode 100644 index 00000000..fafc923b --- /dev/null +++ b/src/runtime/stubs_wasm.go @@ -0,0 +1,16 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// pause sets SP to newsp and pauses the execution of Go's WebAssembly +// code until an event is triggered, or call back into Go. +// +// Note: the epilogue of pause pops 8 bytes from the stack, so when +// returning to the host, the SP is newsp+8. +// If we want to set the SP such that when it calls back into Go, the +// Go function appears to be called from pause's caller's caller, then +// call pause with newsp = internal/runtime/sys.GetCallerSP()-16 (another 8 is +// the return PC pushed to the stack). +func pause(newsp uintptr) diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 10cdcf9c..c3bd5103 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -8,7 +8,7 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -49,7 +49,8 @@ type Frame struct { // File and Line are the file name and line number of the // location in this frame. For non-leaf frames, this will be // the location of a call. These may be the empty string and - // zero, respectively, if not known. + // zero, respectively, if not known. The file name uses + // forward slashes, even on Windows. File string Line int @@ -117,11 +118,16 @@ func (ci *Frames) Next() (frame Frame, more bool) { } f := funcInfo._Func() entry := f.Entry() + // We store the pc of the start of the instruction following + // the instruction in question (the call or the inline mark). + // This is done for historical reasons, and to make FuncForPC + // work correctly for entries in the result of runtime.Callers. + // Decrement to get back to the instruction we care about. + // + // It is not possible to get pc == entry from runtime.Callers, + // but if the caller does provide one, provide best-effort + // results by avoiding backing out of the function entirely. if pc > entry { - // We store the pc of the start of the instruction following - // the instruction in question (the call or the inline mark). - // This is done for historical reasons, and to make FuncForPC - // work correctly for entries in the result of runtime.Callers. pc-- } // It's important that interpret pc non-strictly as cgoTraceback may @@ -462,17 +468,19 @@ type modulehash struct { // To make sure the map isn't collected, we keep a second reference here. var pinnedTypemaps []map[typeOff]*_type -var firstmoduledata moduledata // linker symbol +// aixStaticDataBase (used only on AIX) holds the unrelocated address +// of the data section, set by the linker. +// +// On AIX, an R_ADDR relocation from an RODATA symbol to a DATA symbol +// does not work, as the dynamic loader can change the address of the +// data section, and it is not possible to apply a dynamic relocation +// to RODATA. In order to get the correct address, we need to apply +// the delta between unrelocated and relocated data section addresses. +// aixStaticDataBase is the unrelocated address, and moduledata.data is +// the relocated one. +var aixStaticDataBase uintptr // linker symbol -// lastmoduledatap should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname lastmoduledatap +var firstmoduledata moduledata // linker symbol var lastmoduledatap *moduledata // linker symbol var modulesSlice *[]*moduledata // see activeModules @@ -583,15 +591,6 @@ func moduledataverify() { const debugPcln = false -// moduledataverify1 should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname moduledataverify1 func moduledataverify1(datap *moduledata) { // Check that the pclntab's format is valid. hdr := datap.pcHeader @@ -713,22 +712,24 @@ func (md *moduledata) funcName(nameOff int32) string { return gostringnocopy(&md.funcnametab[nameOff]) } -// FuncForPC returns a *[Func] describing the function that contains the -// given program counter address, or else nil. -// -// If pc represents multiple functions because of inlining, it returns -// the *Func describing the innermost function, but with an entry of -// the outermost function. -// -// For completely unclear reasons, even though they can import runtime, -// some widely used packages access this using linkname. +// Despite being an exported symbol, +// FuncForPC is linknamed by widely used packages. // Notable members of the hall of shame include: // - gitee.com/quant1x/gox // // Do not remove or change the type signature. // See go.dev/issue/67401. // +// Note that this comment is not part of the doc comment. +// //go:linkname FuncForPC + +// FuncForPC returns a *[Func] describing the function that contains the +// given program counter address, or else nil. +// +// If pc represents multiple functions because of inlining, it returns +// the *Func describing the innermost function, but with an entry of +// the outermost function. func FuncForPC(pc uintptr) *Func { f := findfunc(pc) if !f.valid() { @@ -862,7 +863,6 @@ func badFuncInfoEntry(funcInfo) uintptr // findfunc should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/cloudwego/frugal // - github.com/phuslu/log // // Do not remove or change the type signature. @@ -1196,16 +1196,6 @@ func pcdatavalue1(f funcInfo, table uint32, targetpc uintptr, strict bool) int32 } // Like pcdatavalue, but also return the start PC of this PCData value. -// -// pcdatavalue2 should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname pcdatavalue2 func pcdatavalue2(f funcInfo, table uint32, targetpc uintptr) (int32, uintptr) { if table >= f.npcdata { return -1, 0 @@ -1234,16 +1224,6 @@ func funcdata(f funcInfo, i uint8) unsafe.Pointer { } // step advances to the next pc, value pair in the encoded table. -// -// step should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname step func step(p []byte, pc *uintptr, val *int32, first bool) (newp []byte, ok bool) { // For both uvdelta and pcdelta, the common case (~70%) // is that they are a single byte. If so, avoid calling readvarint. @@ -1289,15 +1269,6 @@ type stackmap struct { bytedata [1]byte // bitmaps, each starting on a byte boundary } -// stackmapdata should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname stackmapdata //go:nowritebarrier func stackmapdata(stkmap *stackmap, n int32) bitvector { // Check this invariant only when stackDebug is on at all. diff --git a/src/runtime/symtabinl_test.go b/src/runtime/symtabinl_test.go index 3c7cb2e5..ab58c053 100644 --- a/src/runtime/symtabinl_test.go +++ b/src/runtime/symtabinl_test.go @@ -6,8 +6,8 @@ package runtime import ( "internal/abi" + "internal/runtime/sys" "internal/stringslite" - "runtime/internal/sys" ) func XTestInlineUnwinder(t TestingT) { @@ -107,16 +107,17 @@ func lineNumber() int { // Below here is the test data for XTestInlineUnwinder var tiuStart = lineNumber() // +0 -var tiu1, tiu2, tiu3 int // +1 -func tiuInlined1() { // +2 - tiu1++ // +3 +var tiu2, tiu3 int // +1 +func tiuInlined1(i int) { // +2 + tiu1[i]++ // +3 } // +4 func tiuInlined2() { // +5 - tiuInlined1() // +6 - tiu2++ // +7 + tiuInlined1(1) // +6 + tiu2++ // +7 } // +8 func tiuTest() { // +9 - tiuInlined1() // +10 - tiuInlined2() // +11 - tiu3++ // +12 -} // +13 + tiuInlined1(0) // +10 + tiuInlined2() // +11 + tiu3++ // +12 +} // +13 +var tiu1 [2]int // +14 diff --git a/src/runtime/synctest.go b/src/runtime/synctest.go new file mode 100644 index 00000000..498c3b92 --- /dev/null +++ b/src/runtime/synctest.go @@ -0,0 +1,301 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +// A synctestGroup is a group of goroutines started by synctest.Run. +type synctestGroup struct { + mu mutex + timers timers + now int64 // current fake time + root *g // caller of synctest.Run + waiter *g // caller of synctest.Wait + waiting bool // true if a goroutine is calling synctest.Wait + + // The group is active (not blocked) so long as running > 0 || active > 0. + // + // running is the number of goroutines which are not "durably blocked": + // Goroutines which are either running, runnable, or non-durably blocked + // (for example, blocked in a syscall). + // + // active is used to keep the group from becoming blocked, + // even if all goroutines in the group are blocked. + // For example, park_m can choose to immediately unpark a goroutine after parking it. + // It increments the active count to keep the group active until it has determined + // that the park operation has completed. + total int // total goroutines + running int // non-blocked goroutines + active int // other sources of activity +} + +// changegstatus is called when the non-lock status of a g changes. +// It is never called with a Gscanstatus. +func (sg *synctestGroup) changegstatus(gp *g, oldval, newval uint32) { + // Determine whether this change in status affects the idleness of the group. + // If this isn't a goroutine starting, stopping, durably blocking, + // or waking up after durably blocking, then return immediately without + // locking sg.mu. + // + // For example, stack growth (newstack) will changegstatus + // from _Grunning to _Gcopystack. This is uninteresting to synctest, + // but if stack growth occurs while sg.mu is held, we must not recursively lock. + totalDelta := 0 + wasRunning := true + switch oldval { + case _Gdead: + wasRunning = false + totalDelta++ + case _Gwaiting: + if gp.waitreason.isIdleInSynctest() { + wasRunning = false + } + } + isRunning := true + switch newval { + case _Gdead: + isRunning = false + totalDelta-- + case _Gwaiting: + if gp.waitreason.isIdleInSynctest() { + isRunning = false + } + } + // It's possible for wasRunning == isRunning while totalDelta != 0; + // for example, if a new goroutine is created in a non-running state. + if wasRunning == isRunning && totalDelta == 0 { + return + } + + lock(&sg.mu) + sg.total += totalDelta + if wasRunning != isRunning { + if isRunning { + sg.running++ + } else { + sg.running-- + if raceenabled && newval != _Gdead { + racereleasemergeg(gp, sg.raceaddr()) + } + } + } + if sg.total < 0 { + fatal("total < 0") + } + if sg.running < 0 { + fatal("running < 0") + } + wake := sg.maybeWakeLocked() + unlock(&sg.mu) + if wake != nil { + goready(wake, 0) + } +} + +// incActive increments the active-count for the group. +// A group does not become durably blocked while the active-count is non-zero. +func (sg *synctestGroup) incActive() { + lock(&sg.mu) + sg.active++ + unlock(&sg.mu) +} + +// decActive decrements the active-count for the group. +func (sg *synctestGroup) decActive() { + lock(&sg.mu) + sg.active-- + if sg.active < 0 { + throw("active < 0") + } + wake := sg.maybeWakeLocked() + unlock(&sg.mu) + if wake != nil { + goready(wake, 0) + } +} + +// maybeWakeLocked returns a g to wake if the group is durably blocked. +func (sg *synctestGroup) maybeWakeLocked() *g { + if sg.running > 0 || sg.active > 0 { + return nil + } + // Increment the group active count, since we've determined to wake something. + // The woken goroutine will decrement the count. + // We can't just call goready and let it increment sg.running, + // since we can't call goready with sg.mu held. + // + // Incrementing the active count here is only necessary if something has gone wrong, + // and a goroutine that we considered durably blocked wakes up unexpectedly. + // Two wakes happening at the same time leads to very confusing failure modes, + // so we take steps to avoid it happening. + sg.active++ + if gp := sg.waiter; gp != nil { + // A goroutine is blocked in Wait. Wake it. + return gp + } + // All goroutines in the group are durably blocked, and nothing has called Wait. + // Wake the root goroutine. + return sg.root +} + +func (sg *synctestGroup) raceaddr() unsafe.Pointer { + // Address used to record happens-before relationships created by the group. + // + // Wait creates a happens-before relationship between itself and + // the blocking operations which caused other goroutines in the group to park. + return unsafe.Pointer(sg) +} + +//go:linkname synctestRun internal/synctest.Run +func synctestRun(f func()) { + if debug.asynctimerchan.Load() != 0 { + panic("synctest.Run not supported with asynctimerchan!=0") + } + + gp := getg() + if gp.syncGroup != nil { + panic("synctest.Run called from within a synctest bubble") + } + gp.syncGroup = &synctestGroup{ + total: 1, + running: 1, + root: gp, + } + const synctestBaseTime = 946684800000000000 // midnight UTC 2000-01-01 + gp.syncGroup.now = synctestBaseTime + gp.syncGroup.timers.syncGroup = gp.syncGroup + lockInit(&gp.syncGroup.mu, lockRankSynctest) + lockInit(&gp.syncGroup.timers.mu, lockRankTimers) + defer func() { + gp.syncGroup = nil + }() + + fv := *(**funcval)(unsafe.Pointer(&f)) + newproc(fv) + + sg := gp.syncGroup + lock(&sg.mu) + sg.active++ + for { + if raceenabled { + raceacquireg(gp, gp.syncGroup.raceaddr()) + } + unlock(&sg.mu) + systemstack(func() { + gp.syncGroup.timers.check(gp.syncGroup.now) + }) + gopark(synctestidle_c, nil, waitReasonSynctestRun, traceBlockSynctest, 0) + lock(&sg.mu) + if sg.active < 0 { + throw("active < 0") + } + next := sg.timers.wakeTime() + if next == 0 { + break + } + if next < sg.now { + throw("time went backwards") + } + sg.now = next + } + + total := sg.total + unlock(&sg.mu) + if total != 1 { + panic("deadlock: all goroutines in bubble are blocked") + } + if gp.timer != nil && gp.timer.isFake { + // Verify that we haven't marked this goroutine's sleep timer as fake. + // This could happen if something in Run were to call timeSleep. + throw("synctest root goroutine has a fake timer") + } +} + +func synctestidle_c(gp *g, _ unsafe.Pointer) bool { + lock(&gp.syncGroup.mu) + canIdle := true + if gp.syncGroup.running == 0 && gp.syncGroup.active == 1 { + // All goroutines in the group have blocked or exited. + canIdle = false + } else { + gp.syncGroup.active-- + } + unlock(&gp.syncGroup.mu) + return canIdle +} + +//go:linkname synctestWait internal/synctest.Wait +func synctestWait() { + gp := getg() + if gp.syncGroup == nil { + panic("goroutine is not in a bubble") + } + lock(&gp.syncGroup.mu) + // We use a syncGroup.waiting bool to detect simultaneous calls to Wait rather than + // checking to see if syncGroup.waiter is non-nil. This avoids a race between unlocking + // syncGroup.mu and setting syncGroup.waiter while parking. + if gp.syncGroup.waiting { + unlock(&gp.syncGroup.mu) + panic("wait already in progress") + } + gp.syncGroup.waiting = true + unlock(&gp.syncGroup.mu) + gopark(synctestwait_c, nil, waitReasonSynctestWait, traceBlockSynctest, 0) + + lock(&gp.syncGroup.mu) + gp.syncGroup.active-- + if gp.syncGroup.active < 0 { + throw("active < 0") + } + gp.syncGroup.waiter = nil + gp.syncGroup.waiting = false + unlock(&gp.syncGroup.mu) + + // Establish a happens-before relationship on the activity of the now-blocked + // goroutines in the group. + if raceenabled { + raceacquireg(gp, gp.syncGroup.raceaddr()) + } +} + +func synctestwait_c(gp *g, _ unsafe.Pointer) bool { + lock(&gp.syncGroup.mu) + if gp.syncGroup.running == 0 && gp.syncGroup.active == 0 { + // This shouldn't be possible, since gopark increments active during unlockf. + throw("running == 0 && active == 0") + } + gp.syncGroup.waiter = gp + unlock(&gp.syncGroup.mu) + return true +} + +//go:linkname synctest_acquire internal/synctest.acquire +func synctest_acquire() any { + if sg := getg().syncGroup; sg != nil { + sg.incActive() + return sg + } + return nil +} + +//go:linkname synctest_release internal/synctest.release +func synctest_release(sg any) { + sg.(*synctestGroup).decActive() +} + +//go:linkname synctest_inBubble internal/synctest.inBubble +func synctest_inBubble(sg any, f func()) { + gp := getg() + if gp.syncGroup != nil { + panic("goroutine is already bubbled") + } + gp.syncGroup = sg.(*synctestGroup) + defer func() { + gp.syncGroup = nil + }() + f() +} diff --git a/src/runtime/synctest_test.go b/src/runtime/synctest_test.go new file mode 100644 index 00000000..0fdd032f --- /dev/null +++ b/src/runtime/synctest_test.go @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "testing" +) + +func TestSynctest(t *testing.T) { + output := runTestProg(t, "testsynctest", "") + want := "success\n" + if output != want { + t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want) + } +} diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 1e4b2ac7..5c769a71 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -571,6 +571,15 @@ func pthread_cond_signal(c *pthreadcond) int32 { } func pthread_cond_signal_trampoline() +//go:nosplit +//go:cgo_unsafe_args +func arc4random_buf(p unsafe.Pointer, n int32) { + // arc4random_buf() never fails, per its man page, so it's safe to ignore the return value. + libcCall(unsafe.Pointer(abi.FuncPCABI0(arc4random_buf_trampoline)), unsafe.Pointer(&p)) + KeepAlive(p) +} +func arc4random_buf_trampoline() + // Not used on Darwin, but must be defined. func exitThread(wait *atomic.Uint32) { throw("exitThread") @@ -691,6 +700,7 @@ func proc_regionfilename_trampoline() //go:cgo_import_dynamic libc_pthread_cond_wait pthread_cond_wait "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_timedwait_relative_np pthread_cond_timedwait_relative_np "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_signal pthread_cond_signal "/usr/lib/libSystem.B.dylib" +//go:cgo_import_dynamic libc_arc4random_buf arc4random_buf "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_notify_is_valid_token notify_is_valid_token "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_xpc_date_create_from_current xpc_date_create_from_current "/usr/lib/libSystem.B.dylib" diff --git a/src/runtime/sys_darwin_amd64.s b/src/runtime/sys_darwin_amd64.s index 01992d59..acf24d2f 100644 --- a/src/runtime/sys_darwin_amd64.s +++ b/src/runtime/sys_darwin_amd64.s @@ -500,6 +500,12 @@ TEXT runtime·osinit_hack_trampoline(SB),NOSPLIT,$0 CALL libc_xpc_date_create_from_current(SB) RET +TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0 + MOVL 8(DI), SI // arg 2 nbytes + MOVQ 0(DI), DI // arg 1 buf + CALL libc_arc4random_buf(SB) + RET + // syscall calls a function in libc on behalf of the syscall package. // syscall takes a pointer to a struct like: // struct { diff --git a/src/runtime/sys_darwin_arm64.s b/src/runtime/sys_darwin_arm64.s index 32d1f95d..788fdf87 100644 --- a/src/runtime/sys_darwin_arm64.s +++ b/src/runtime/sys_darwin_arm64.s @@ -475,6 +475,12 @@ TEXT runtime·osinit_hack_trampoline(SB),NOSPLIT,$0 BL libc_xpc_date_create_from_current(SB) RET +TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0 + MOVW 8(R0), R1 // arg 2 nbytes + MOVD 0(R0), R0 // arg 1 buf + BL libc_arc4random_buf(SB) + RET + // syscall calls a function in libc on behalf of the syscall package. // syscall takes a pointer to a struct like: // struct { diff --git a/src/runtime/sys_libc.go b/src/runtime/sys_libc.go index 0c6f13ca..72d89915 100644 --- a/src/runtime/sys_libc.go +++ b/src/runtime/sys_libc.go @@ -6,7 +6,10 @@ package runtime -import "unsafe" +import ( + "internal/runtime/sys" + "unsafe" +) // Call fn with arg as its argument. Return what fn returns. // fn is the raw pc value of the entry point of the desired function. @@ -23,10 +26,10 @@ func libcCall(fn, arg unsafe.Pointer) int32 { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { // Make sure we don't reset libcallsp. This makes // libcCall reentrant; We remember the g/pc/sp for the diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index b6c64dc0..941f70b0 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -704,3 +704,36 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8 SYSCALL MOVQ AX, ret+0(FP) RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVQ SI, R8 // stateSize + MOVL CX, DX // flags + MOVQ DI, CX // state + MOVQ BX, SI // length + MOVQ AX, DI // buf + + MOVQ SP, R12 + + MOVQ runtime·vdsoGetrandomSym(SB), AX + MOVQ g_m(R14), BX + + MOVQ m_vdsoPC(BX), R9 + MOVQ R9, 0(SP) + MOVQ m_vdsoSP(BX), R9 + MOVQ R9, 8(SP) + LEAQ buf+0(FP), R9 + MOVQ R9, m_vdsoSP(BX) + MOVQ -8(R9), R9 + MOVQ R9, m_vdsoPC(BX) + + ANDQ $~15, SP + + CALL AX + + MOVQ R12, SP + MOVQ 8(SP), R9 + MOVQ R9, m_vdsoSP(BX) + MOVQ 0(SP), R9 + MOVQ R9, m_vdsoPC(BX) + RET diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s index 51c87bea..7a81d547 100644 --- a/src/runtime/sys_linux_arm64.s +++ b/src/runtime/sys_linux_arm64.s @@ -785,3 +785,48 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8 SVC MOVD R0, ret+0(FP) RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVD RSP, R20 + + MOVD runtime·vdsoGetrandomSym(SB), R8 + MOVD g_m(g), R21 + + MOVD m_vdsoPC(R21), R9 + MOVD R9, 8(RSP) + MOVD m_vdsoSP(R21), R9 + MOVD R9, 16(RSP) + MOVD LR, m_vdsoPC(R21) + MOVD $buf-8(FP), R9 + MOVD R9, m_vdsoSP(R21) + + MOVD RSP, R9 + BIC $15, R9 + MOVD R9, RSP + + MOVBU runtime·iscgo(SB), R9 + CBNZ R9, nosaveg + MOVD m_gsignal(R21), R9 + CBZ R9, nosaveg + CMP g, R9 + BEQ nosaveg + MOVD (g_stack+stack_lo)(R9), R22 + MOVD g, (R22) + + BL (R8) + + MOVD ZR, (R22) + B restore + +nosaveg: + BL (R8) + +restore: + MOVD R20, RSP + MOVD 16(RSP), R1 + MOVD R1, m_vdsoSP(R21) + MOVD 8(RSP), R1 + MOVD R1, m_vdsoPC(R21) + NOP R0 // Satisfy go vet, since the return value comes from the vDSO function. + RET diff --git a/src/runtime/sys_linux_loong64.s b/src/runtime/sys_linux_loong64.s index eba8e1f2..57cee99d 100644 --- a/src/runtime/sys_linux_loong64.s +++ b/src/runtime/sys_linux_loong64.s @@ -431,12 +431,9 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 JAL (R20) RET +// Called from c-abi, R4: sig, R5: info, R6: cxt // func sigtramp(signo, ureg, ctxt unsafe.Pointer) TEXT runtime·sigtramp(SB),NOSPLIT|TOPFRAME,$168 - MOVW R4, (1*8)(R3) - MOVV R5, (2*8)(R3) - MOVV R6, (3*8)(R3) - // Save callee-save registers in the case of signal forwarding. // Please refer to https://golang.org/issue/31827 . SAVE_R22_TO_R31((4*8)) @@ -444,12 +441,13 @@ TEXT runtime·sigtramp(SB),NOSPLIT|TOPFRAME,$168 // this might be called in external code context, // where g is not set. - MOVB runtime·iscgo(SB), R4 - BEQ R4, 2(PC) + MOVB runtime·iscgo(SB), R7 + BEQ R7, 2(PC) JAL runtime·load_g(SB) - MOVV $runtime·sigtrampgo(SB), R4 - JAL (R4) + // R5 and R6 already contain info and ctx, respectively. + MOVV $runtime·sigtrampgo(SB), R7 + JAL (R7) // Restore callee-save registers. RESTORE_R22_TO_R31((4*8)) @@ -657,3 +655,46 @@ TEXT runtime·socket(SB),$0-20 MOVV R0, 2(R0) // unimplemented, only needed for android; declared in stubs_linux.go MOVW R0, ret+16(FP) // for vet RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVV R3, R23 + + MOVV runtime·vdsoGetrandomSym(SB), R12 + + MOVV g_m(g), R24 + + MOVV m_vdsoPC(R24), R13 + MOVV R13, 8(R3) + MOVV m_vdsoSP(R24), R13 + MOVV R13, 16(R3) + MOVV R1, m_vdsoPC(R24) + MOVV $buf-8(FP), R13 + MOVV R13, m_vdsoSP(R24) + + AND $~15, R3 + + MOVBU runtime·iscgo(SB), R13 + BNE R13, nosaveg + MOVV m_gsignal(R24), R13 + BEQ R13, nosaveg + BEQ g, R13, nosaveg + MOVV (g_stack+stack_lo)(R13), R25 + MOVV g, (R25) + + JAL (R12) + + MOVV R0, (R25) + JMP restore + +nosaveg: + JAL (R12) + +restore: + MOVV R23, R3 + MOVV 16(R3), R25 + MOVV R25, m_vdsoSP(R24) + MOVV 8(R3), R25 + MOVV R25, m_vdsoPC(R24) + NOP R4 // Satisfy go vet, since the return value comes from the vDSO function. + RET diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index ba4988b7..8735b932 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -757,3 +757,53 @@ TEXT runtime·socket(SB),$0-20 MOVD R0, 0(R0) // unimplemented, only needed for android; declared in stubs_linux.go MOVW R0, ret+16(FP) // for vet RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVD R1, R15 + + MOVD runtime·vdsoGetrandomSym(SB), R12 + MOVD R12, CTR + MOVD g_m(g), R21 + + MOVD m_vdsoPC(R21), R22 + MOVD R22, 32(R1) + MOVD m_vdsoSP(R21), R22 + MOVD R22, 40(R1) + MOVD LR, m_vdsoPC(R21) + MOVD $buf-FIXED_FRAME(FP), R22 + MOVD R22, m_vdsoSP(R21) + + RLDICR $0, R1, $59, R1 + + MOVBZ runtime·iscgo(SB), R22 + CMP R22, $0 + BNE nosaveg + MOVD m_gsignal(R21), R22 + CMP R22, $0 + BEQ nosaveg + CMP R22, g + BEQ nosaveg + MOVD (g_stack+stack_lo)(R22), R22 + MOVD g, (R22) + + BL (CTR) + + MOVD $0, (R22) + JMP restore + +nosaveg: + BL (CTR) + +restore: + MOVD $0, R0 + MOVD R15, R1 + MOVD 40(R1), R22 + MOVD R22, m_vdsoSP(R21) + MOVD 32(R1), R22 + MOVD R22, m_vdsoPC(R21) + + BVC out + NEG R3, R3 +out: + RET diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s index ffec2b5b..0d4fb3b5 100644 --- a/src/runtime/sys_linux_riscv64.s +++ b/src/runtime/sys_linux_riscv64.s @@ -51,8 +51,7 @@ #define SYS_write 64 // func exit(code int32) -TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 - MOVW code+0(FP), A0 +TEXT runtime·exit(SB),NOSPLIT,$0 MOV $SYS_exit_group, A7 ECALL RET @@ -95,23 +94,15 @@ TEXT runtime·closefd(SB),NOSPLIT|NOFRAME,$0-12 RET // func write1(fd uintptr, p unsafe.Pointer, n int32) int32 -TEXT runtime·write1(SB),NOSPLIT|NOFRAME,$0-28 - MOV fd+0(FP), A0 - MOV p+8(FP), A1 - MOVW n+16(FP), A2 +TEXT runtime·write1(SB),NOSPLIT,$0 MOV $SYS_write, A7 ECALL - MOVW A0, ret+24(FP) RET // func read(fd int32, p unsafe.Pointer, n int32) int32 -TEXT runtime·read(SB),NOSPLIT|NOFRAME,$0-28 - MOVW fd+0(FP), A0 - MOV p+8(FP), A1 - MOVW n+16(FP), A2 +TEXT runtime·read(SB),NOSPLIT,$0 MOV $SYS_read, A7 ECALL - MOVW A0, ret+24(FP) RET // func pipe2(flags int32) (r, w int32, errno int32) @@ -140,10 +131,9 @@ TEXT runtime·usleep(SB),NOSPLIT,$24-4 RET // func gettid() uint32 -TEXT runtime·gettid(SB),NOSPLIT,$0-4 +TEXT runtime·gettid(SB),NOSPLIT,$0 MOV $SYS_gettid, A7 ECALL - MOVW A0, ret+0(FP) RET // func raise(sig uint32) @@ -167,67 +157,45 @@ TEXT runtime·raiseproc(SB),NOSPLIT|NOFRAME,$0 RET // func getpid() int -TEXT ·getpid(SB),NOSPLIT|NOFRAME,$0-8 +TEXT ·getpid(SB),NOSPLIT,$0 MOV $SYS_getpid, A7 ECALL - MOV A0, ret+0(FP) RET // func tgkill(tgid, tid, sig int) -TEXT ·tgkill(SB),NOSPLIT|NOFRAME,$0-24 - MOV tgid+0(FP), A0 - MOV tid+8(FP), A1 - MOV sig+16(FP), A2 +TEXT ·tgkill(SB),NOSPLIT,$0 MOV $SYS_tgkill, A7 ECALL RET // func setitimer(mode int32, new, old *itimerval) -TEXT runtime·setitimer(SB),NOSPLIT|NOFRAME,$0-24 - MOVW mode+0(FP), A0 - MOV new+8(FP), A1 - MOV old+16(FP), A2 +TEXT runtime·setitimer(SB),NOSPLIT,$0 MOV $SYS_setitimer, A7 ECALL RET // func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32 -TEXT runtime·timer_create(SB),NOSPLIT,$0-28 - MOVW clockid+0(FP), A0 - MOV sevp+8(FP), A1 - MOV timerid+16(FP), A2 +TEXT runtime·timer_create(SB),NOSPLIT,$0 MOV $SYS_timer_create, A7 ECALL - MOVW A0, ret+24(FP) RET // func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32 -TEXT runtime·timer_settime(SB),NOSPLIT,$0-28 - MOVW timerid+0(FP), A0 - MOVW flags+4(FP), A1 - MOV new+8(FP), A2 - MOV old+16(FP), A3 +TEXT runtime·timer_settime(SB),NOSPLIT,$0 MOV $SYS_timer_settime, A7 ECALL - MOVW A0, ret+24(FP) RET // func timer_delete(timerid int32) int32 -TEXT runtime·timer_delete(SB),NOSPLIT,$0-12 - MOVW timerid+0(FP), A0 +TEXT runtime·timer_delete(SB),NOSPLIT,$0 MOV $SYS_timer_delete, A7 ECALL - MOVW A0, ret+8(FP) RET // func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 -TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28 - MOV addr+0(FP), A0 - MOV n+8(FP), A1 - MOV dst+16(FP), A2 +TEXT runtime·mincore(SB),NOSPLIT,$0 MOV $SYS_mincore, A7 ECALL - MOVW A0, ret+24(FP) RET // func walltime() (sec int64, nsec int32) @@ -393,14 +361,9 @@ TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28 RET // func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 -TEXT runtime·rt_sigaction(SB),NOSPLIT|NOFRAME,$0-36 - MOV sig+0(FP), A0 - MOV new+8(FP), A1 - MOV old+16(FP), A2 - MOV size+24(FP), A3 +TEXT runtime·rt_sigaction(SB),NOSPLIT,$0 MOV $SYS_rt_sigaction, A7 ECALL - MOVW A0, ret+32(FP) RET // func sigfwd(fn uintptr, sig uint32, info *siginfo, ctx unsafe.Pointer) @@ -466,26 +429,15 @@ TEXT runtime·munmap(SB),NOSPLIT|NOFRAME,$0 RET // func madvise(addr unsafe.Pointer, n uintptr, flags int32) -TEXT runtime·madvise(SB),NOSPLIT|NOFRAME,$0 - MOV addr+0(FP), A0 - MOV n+8(FP), A1 - MOVW flags+16(FP), A2 +TEXT runtime·madvise(SB),NOSPLIT,$0 MOV $SYS_madvise, A7 ECALL - MOVW A0, ret+24(FP) RET // func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 -TEXT runtime·futex(SB),NOSPLIT|NOFRAME,$0 - MOV addr+0(FP), A0 - MOVW op+8(FP), A1 - MOVW val+12(FP), A2 - MOV ts+16(FP), A3 - MOV addr2+24(FP), A4 - MOVW val3+32(FP), A5 +TEXT runtime·futex(SB),NOSPLIT,$0 MOV $SYS_futex, A7 ECALL - MOVW A0, ret+40(FP) RET // func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32 @@ -559,26 +511,21 @@ TEXT runtime·sigaltstack(SB),NOSPLIT|NOFRAME,$0 RET // func osyield() -TEXT runtime·osyield(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·osyield(SB),NOSPLIT,$0 MOV $SYS_sched_yield, A7 ECALL RET // func sched_getaffinity(pid, len uintptr, buf *uintptr) int32 -TEXT runtime·sched_getaffinity(SB),NOSPLIT|NOFRAME,$0 - MOV pid+0(FP), A0 - MOV len+8(FP), A1 - MOV buf+16(FP), A2 +TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0 MOV $SYS_sched_getaffinity, A7 ECALL - MOV A0, ret+24(FP) RET // func sbrk0() uintptr -TEXT runtime·sbrk0(SB),NOSPLIT,$0-8 +TEXT runtime·sbrk0(SB),NOSPLIT,$0 // Implemented as brk(NULL). - MOV $0, A0 + MOV ZERO, A0 MOV $SYS_brk, A7 ECALL - MOVW A0, ret+0(FP) RET diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index adf5612c..7da4a527 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -604,3 +604,53 @@ TEXT runtime·socket(SB),$0-20 MOVD $0, 2(R0) // unimplemented, only needed for android; declared in stubs_linux.go MOVW R0, ret+16(FP) RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVD buf+0(FP), R2 + MOVD length+8(FP), R3 + MOVW flags+16(FP), R4 + MOVD state+24(FP), R5 + MOVD stateSize+32(FP), R6 + + MOVD R15, R7 + + MOVD runtime·vdsoGetrandomSym(SB), R1 + MOVD g_m(g), R9 + + MOVD m_vdsoPC(R9), R12 + MOVD R12, 8(R15) + MOVD m_vdsoSP(R9), R12 + MOVD R12, 16(R15) + MOVD R14, m_vdsoPC(R9) + MOVD $buf+0(FP), R12 + MOVD R12, m_vdsoSP(R9) + + SUB $160, R15 + MOVD $~7, R12 + AND R12, R15 + + MOVB runtime·iscgo(SB), R12 + CMPBNE R12, $0, nosaveg + MOVD m_gsignal(R9), R12 + CMPBEQ R12, $0, nosaveg + CMPBEQ g, R12, nosaveg + MOVD (g_stack+stack_lo)(R12), R12 + MOVD g, (R12) + + BL R1 + + MOVD $0, (R12) + JMP restore + +nosaveg: + BL R1 + +restore: + MOVD R7, R15 + MOVD 16(R15), R12 + MOVD R12, m_vdsoSP(R9) + MOVD 8(R15), R12 + MOVD R12, m_vdsoPC(R9) + MOVD R2, ret+40(FP) + RET diff --git a/src/runtime/sys_wasm.go b/src/runtime/sys_wasm.go index 27f9432b..6b40a8d3 100644 --- a/src/runtime/sys_wasm.go +++ b/src/runtime/sys_wasm.go @@ -6,7 +6,7 @@ package runtime import ( "internal/goarch" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -34,3 +34,17 @@ func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { buf.pc = uintptr(fn) buf.ctxt = ctxt } + +func notInitialized() // defined in assembly, call notInitialized1 + +// Called if a wasmexport function is called before runtime initialization +// +//go:nosplit +func notInitialized1() { + writeErrStr("runtime: wasmexport function called before runtime initialization\n") + if isarchive || islibrary { + writeErrStr("\tcall _initialize first\n") + } else { + writeErrStr("\tcall _start first\n") + } +} diff --git a/src/runtime/sys_wasm.s b/src/runtime/sys_wasm.s index 1e73adad..b7965ec3 100644 --- a/src/runtime/sys_wasm.s +++ b/src/runtime/sys_wasm.s @@ -86,6 +86,12 @@ TEXT runtime·exitThread(SB), NOSPLIT, $0-0 TEXT runtime·osyield(SB), NOSPLIT, $0-0 UNDEF +TEXT runtime·currentMemory(SB), NOSPLIT, $0 + Get SP + CurrentMemory + I32Store ret+0(FP) + RET + TEXT runtime·growMemory(SB), NOSPLIT, $0 Get SP I32Load pages+0(FP) diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index d194899d..99f33cf0 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -205,6 +205,7 @@ TEXT runtime·read_tls_fallback(SB),NOSPLIT,$0 RET TEXT runtime·nanotime1(SB),NOSPLIT,$0-8 + MOVW $_INTERRUPT_TIME, R3 loop: MOVW time_hi1(R3), R1 DMB MB_ISH diff --git a/src/runtime/syscall_windows_test.go b/src/runtime/syscall_windows_test.go index 2db5b61a..53f71108 100644 --- a/src/runtime/syscall_windows_test.go +++ b/src/runtime/syscall_windows_test.go @@ -673,7 +673,7 @@ func TestWindowsStackMemory(t *testing.T) { if err != nil { t.Fatalf("Failed to read stack usage: %v", err) } - if expected, got := 100<<10, stackUsage; got > expected { + if expected, got := 128<<10, stackUsage; got > expected { t.Fatalf("expected < %d bytes of memory per thread, got %d", expected, got) } } @@ -1111,12 +1111,6 @@ func TestDLLPreloadMitigation(t *testing.T) { tmpdir := t.TempDir() - dir0, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - defer os.Chdir(dir0) - const src = ` #include #include @@ -1127,7 +1121,7 @@ uintptr_t cfunc(void) { } ` srcname := "nojack.c" - err = os.WriteFile(filepath.Join(tmpdir, srcname), []byte(src), 0) + err := os.WriteFile(filepath.Join(tmpdir, srcname), []byte(src), 0) if err != nil { t.Fatal(err) } @@ -1148,7 +1142,7 @@ uintptr_t cfunc(void) { // ("nojack.dll") Think of this as the user double-clicking an // installer from their Downloads directory where a browser // silently downloaded some malicious DLLs. - os.Chdir(tmpdir) + t.Chdir(tmpdir) // First before we can load a DLL from the current directory, // loading it only as "nojack.dll", without an absolute path. diff --git a/src/runtime/testdata/testprogcgo/callback.go b/src/runtime/testdata/testprogcgo/callback.go index 319572fe..39993f13 100644 --- a/src/runtime/testdata/testprogcgo/callback.go +++ b/src/runtime/testdata/testprogcgo/callback.go @@ -38,10 +38,21 @@ import ( func init() { register("CgoCallbackGC", CgoCallbackGC) + register("CgoToGoCallGoexit", CgoToGoCallGoexit) } +func CgoToGoCallGoexit() { + goexit = true + C.foo() +} + +var goexit = false + //export go_callback func go_callback() { + if goexit { + runtime.Goexit() + } if e := extraMInUse.Load(); e == 0 { fmt.Printf("in callback extraMInUse got %d want >0\n", e) os.Exit(1) diff --git a/src/runtime/testdata/testprogcgo/cgonocallback.go b/src/runtime/testdata/testprogcgo/cgonocallback.go index c13bf271..8cbbfd19 100644 --- a/src/runtime/testdata/testprogcgo/cgonocallback.go +++ b/src/runtime/testdata/testprogcgo/cgonocallback.go @@ -8,7 +8,8 @@ package main // But it do callback to go in this test, Go should crash here. /* -// TODO(#56378): #cgo nocallback runCShouldNotCallback +#cgo nocallback runCShouldNotCallback + extern void runCShouldNotCallback(); */ import "C" diff --git a/src/runtime/testdata/testprogcgo/cgonoescape.go b/src/runtime/testdata/testprogcgo/cgonoescape.go index f5eebac6..2b7c7a9c 100644 --- a/src/runtime/testdata/testprogcgo/cgonoescape.go +++ b/src/runtime/testdata/testprogcgo/cgonoescape.go @@ -13,7 +13,8 @@ package main // 2. less than 100 new allocated heap objects after invoking withoutNoEscape 100 times. /* -// TODO(#56378): #cgo noescape runCWithNoEscape +#cgo noescape runCWithNoEscape +#cgo nocallback runCWithNoEscape void runCWithNoEscape(void *p) { } diff --git a/src/runtime/testdata/testprogcgo/issue63739.go b/src/runtime/testdata/testprogcgo/issue63739.go new file mode 100644 index 00000000..dbe37b6d --- /dev/null +++ b/src/runtime/testdata/testprogcgo/issue63739.go @@ -0,0 +1,59 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This is for issue #63739. +// Ensure that parameters are kept alive until the end of the C call. If not, +// then a stack copy at just the right time while calling into C might think +// that any stack pointers are not alive and fail to update them, causing the C +// function to see the old, no longer correct, pointer values. + +/* +int add_from_multiple_pointers(int *a, int *b, int *c) { + *a = *a + 1; + *b = *b + 1; + *c = *c + 1; + return *a + *b + *c; +} +#cgo noescape add_from_multiple_pointers +#cgo nocallback add_from_multiple_pointers +*/ +import "C" + +import ( + "fmt" +) + +const ( + maxStack = 1024 +) + +func init() { + register("CgoEscapeWithMultiplePointers", CgoEscapeWithMultiplePointers) +} + +func CgoEscapeWithMultiplePointers() { + stackGrow(maxStack) + fmt.Println("OK") +} + +//go:noinline +func testCWithMultiplePointers() { + var a C.int = 1 + var b C.int = 2 + var c C.int = 3 + v := C.add_from_multiple_pointers(&a, &b, &c) + if v != 9 || a != 2 || b != 3 || c != 4 { + fmt.Printf("%d + %d + %d != %d\n", a, b, c, v) + } +} + +func stackGrow(n int) { + if n == 0 { + return + } + testCWithMultiplePointers() + stackGrow(n - 1) +} diff --git a/src/runtime/testdata/testprogcgo/threadprof.go b/src/runtime/testdata/testprogcgo/threadprof.go index 00b511d2..2d8c0f6d 100644 --- a/src/runtime/testdata/testprogcgo/threadprof.go +++ b/src/runtime/testdata/testprogcgo/threadprof.go @@ -36,10 +36,10 @@ __attribute__((constructor)) void issue9456() { } } -void **nullptr; +void **nullpointer; void *crash(void *p) { - *nullptr = p; + *nullpointer = p; return 0; } diff --git a/src/runtime/testdata/testsynctest/main.go b/src/runtime/testdata/testsynctest/main.go new file mode 100644 index 00000000..d2cbc992 --- /dev/null +++ b/src/runtime/testdata/testsynctest/main.go @@ -0,0 +1,67 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "internal/synctest" + "runtime" + "runtime/metrics" +) + +// This program ensures system goroutines (GC workers, finalizer goroutine) +// started from within a synctest bubble do not participate in that bubble. +// +// To ensure none of these goroutines start before synctest.Run, +// it must have no dependencies on packages which may start system goroutines. +// This includes the os package, which creates finalizers at init time. + +func numGCCycles() uint64 { + samples := []metrics.Sample{{Name: "/gc/cycles/total:gc-cycles"}} + metrics.Read(samples) + if samples[0].Value.Kind() == metrics.KindBad { + panic("metric not supported") + } + return samples[0].Value.Uint64() +} + +func main() { + synctest.Run(func() { + // Start the finalizer goroutine. + p := new(int) + runtime.SetFinalizer(p, func(*int) {}) + + startingCycles := numGCCycles() + ch1 := make(chan *int) + ch2 := make(chan *int) + defer close(ch1) + go func() { + for i := range ch1 { + v := *i + 1 + ch2 <- &v + } + }() + for { + // Make a lot of short-lived allocations to get the GC working. + for i := 0; i < 1000; i++ { + v := new(int) + *v = i + // Set finalizers on these values, just for added stress. + runtime.SetFinalizer(v, func(*int) {}) + ch1 <- v + <-ch2 + } + + // If we've improperly put a GC goroutine into the synctest group, + // this Wait is going to hang. + synctest.Wait() + + // End the test after a couple of GC cycles have passed. + if numGCCycles()-startingCycles > 1 { + break + } + } + }) + println("success") +} diff --git a/src/runtime/time.go b/src/runtime/time.go index 7b344a34..c22d39c0 100644 --- a/src/runtime/time.go +++ b/src/runtime/time.go @@ -9,10 +9,29 @@ package runtime import ( "internal/abi" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) +//go:linkname time_runtimeNow time.runtimeNow +func time_runtimeNow() (sec int64, nsec int32, mono int64) { + if sg := getg().syncGroup; sg != nil { + sec = sg.now / (1000 * 1000 * 1000) + nsec = int32(sg.now % (1000 * 1000 * 1000)) + return sec, nsec, sg.now + } + return time_now() +} + +//go:linkname time_runtimeNano time.runtimeNano +func time_runtimeNano() int64 { + gp := getg() + if gp.syncGroup != nil { + return gp.syncGroup.now + } + return nanotime() +} + // A timer is a potentially repeating trigger for calling t.f(t.arg, t.seq). // Timers are allocated by client code, often as part of other data structures. // Each P has a heap of pointers to timers that it manages. @@ -29,6 +48,7 @@ type timer struct { astate atomic.Uint8 // atomic copy of state bits at last unlock state uint8 // state bits isChan bool // timer has a channel; immutable; can be read without lock + isFake bool // timer is using fake time; immutable; can be read without lock blocked uint32 // number of goroutines blocked on timer's channel @@ -39,7 +59,7 @@ type timer struct { // The arg and seq are client-specified opaque arguments passed back to f. // When used from netpoll, arg and seq have meanings defined by netpoll // and are completely opaque to this code; in that context, seq is a sequence - // number to recognize and squech stale function invocations. + // number to recognize and squelch stale function invocations. // When used from package time, arg is a channel (for After, NewTicker) // or the function to call (for AfterFunc) and seq is unused (0). // @@ -125,6 +145,8 @@ type timers struct { // heap[i].when over timers with the timerModified bit set. // If minWhenModified = 0, it means there are no timerModified timers in the heap. minWhenModified atomic.Int64 + + syncGroup *synctestGroup } type timerWhen struct { @@ -290,14 +312,31 @@ func timeSleep(ns int64) { if t == nil { t = new(timer) t.init(goroutineReady, gp) + if gp.syncGroup != nil { + t.isFake = true + } gp.timer = t } - when := nanotime() + ns + var now int64 + if sg := gp.syncGroup; sg != nil { + now = sg.now + } else { + now = nanotime() + } + when := now + ns if when < 0 { // check for overflow. when = maxWhen } gp.sleepWhen = when - gopark(resetForSleep, nil, waitReasonSleep, traceBlockSleep, 1) + if t.isFake { + // Call timer.reset in this goroutine, since it's the one in a syncGroup. + // We don't need to worry about the timer function running before the goroutine + // is parked, because time won't advance until we park. + resetForSleep(gp, nil) + gopark(nil, nil, waitReasonSleep, traceBlockSleep, 1) + } else { + gopark(resetForSleep, nil, waitReasonSleep, traceBlockSleep, 1) + } } // resetForSleep is called after the goroutine is parked for timeSleep. @@ -337,6 +376,9 @@ func newTimer(when, period int64, f func(arg any, seq uintptr, delay int64), arg throw("invalid timer channel: no capacity") } } + if gr := getg().syncGroup; gr != nil { + t.isFake = true + } t.modify(when, period, f, arg, 0) t.init = true return t @@ -347,6 +389,9 @@ func newTimer(when, period int64, f func(arg any, seq uintptr, delay int64), arg // //go:linkname stopTimer time.stopTimer func stopTimer(t *timeTimer) bool { + if t.isFake && getg().syncGroup == nil { + panic("stop of synctest timer from outside bubble") + } return t.stop() } @@ -359,6 +404,9 @@ func resetTimer(t *timeTimer, when, period int64) bool { if raceenabled { racerelease(unsafe.Pointer(&t.timer)) } + if t.isFake && getg().syncGroup == nil { + panic("reset of synctest timer from outside bubble") + } return t.reset(when, period) } @@ -582,7 +630,7 @@ func (t *timer) modify(when, period int64, f func(arg any, seq uintptr, delay in // t must be locked. func (t *timer) needsAdd() bool { assertLockHeld(&t.mu) - need := t.state&timerHeaped == 0 && t.when > 0 && (!t.isChan || t.blocked > 0) + need := t.state&timerHeaped == 0 && t.when > 0 && (!t.isChan || t.isFake || t.blocked > 0) if need { t.trace("needsAdd+") } else { @@ -620,7 +668,16 @@ func (t *timer) maybeAdd() { // Calling acquirem instead of using getg().m makes sure that // we end up locking and inserting into the current P's timers. mp := acquirem() - ts := &mp.p.ptr().timers + var ts *timers + if t.isFake { + sg := getg().syncGroup + if sg == nil { + throw("invalid timer: fake time but no syncgroup") + } + ts = &sg.timers + } else { + ts = &mp.p.ptr().timers + } ts.lock() ts.cleanHead() t.lock() @@ -1071,6 +1128,16 @@ func (t *timer) unlockAndRun(now int64) { ts.unlock() } + if ts != nil && ts.syncGroup != nil { + // Temporarily use the timer's synctest group for the G running this timer. + gp := getg() + if gp.syncGroup != nil { + throw("unexpected syncgroup set") + } + gp.syncGroup = ts.syncGroup + ts.syncGroup.changegstatus(gp, _Gdead, _Grunning) + } + if !async && t.isChan { // For a timer channel, we want to make sure that no stale sends // happen after a t.stop or t.modify, but we cannot hold t.mu @@ -1112,6 +1179,12 @@ func (t *timer) unlockAndRun(now int64) { unlock(&t.sendLock) } + if ts != nil && ts.syncGroup != nil { + gp := getg() + ts.syncGroup.changegstatus(gp, _Grunning, _Gdead) + gp.syncGroup = nil + } + if ts != nil { ts.lock() } @@ -1297,6 +1370,24 @@ func badTimer() { // to send a value to its associated channel. If so, it does. // The timer must not be locked. func (t *timer) maybeRunChan() { + if t.isFake { + t.lock() + var timerGroup *synctestGroup + if t.ts != nil { + timerGroup = t.ts.syncGroup + } + t.unlock() + sg := getg().syncGroup + if sg == nil { + panic(plainError("synctest timer accessed from outside bubble")) + } + if timerGroup != nil && sg != timerGroup { + panic(plainError("timer moved between synctest bubbles")) + } + // No need to do anything here. + // synctest.Run will run the timer when it advances its fake clock. + return + } if t.astate.Load()&timerHeaped != 0 { // If the timer is in the heap, the ordinary timer code // is in charge of sending when appropriate. @@ -1323,6 +1414,9 @@ func (t *timer) maybeRunChan() { // adding it if needed. func blockTimerChan(c *hchan) { t := c.timer + if t.isFake { + return + } t.lock() t.trace("blockTimerChan") if !t.isChan { @@ -1360,6 +1454,9 @@ func blockTimerChan(c *hchan) { // blocked on it anymore. func unblockTimerChan(c *hchan) { t := c.timer + if t.isFake { + return + } t.lock() t.trace("unblockTimerChan") if !t.isChan || t.blocked == 0 { diff --git a/src/runtime/time_linux_amd64.s b/src/runtime/time_linux_amd64.s index 1416d232..fa9561b2 100644 --- a/src/runtime/time_linux_amd64.s +++ b/src/runtime/time_linux_amd64.s @@ -10,7 +10,7 @@ #define SYS_clock_gettime 228 -// func time.now() (sec int64, nsec int32, mono int64) +// func now() (sec int64, nsec int32, mono int64) TEXT time·now(SB),NOSPLIT,$16-24 MOVQ SP, R12 // Save old SP; R12 unchanged by C code. diff --git a/src/runtime/trace.go b/src/runtime/trace.go index adf7b095..bc2978bb 100644 --- a/src/runtime/trace.go +++ b/src/runtime/trace.go @@ -524,10 +524,11 @@ func traceAdvance(stopTrace bool) { // trace.lock needed for traceBufFlush, but also to synchronize // with traceThreadDestroy, which flushes both buffers unconditionally. lock(&trace.lock) - bufp := &mp.trace.buf[gen%2] - if *bufp != nil { - traceBufFlush(*bufp, gen) - *bufp = nil + for exp, buf := range mp.trace.buf[gen%2] { + if buf != nil { + traceBufFlush(buf, gen) + mp.trace.buf[gen%2][exp] = nil + } } unlock(&trace.lock) diff --git a/src/runtime/trace_cgo_test.go b/src/runtime/trace_cgo_test.go index f0db3b7f..871698f8 100644 --- a/src/runtime/trace_cgo_test.go +++ b/src/runtime/trace_cgo_test.go @@ -94,10 +94,9 @@ func mustFindLogV2(t *testing.T, trc io.Reader, category string) trace.Event { // dumpStack returns e.Stack() as a string. func dumpStackV2(e *trace.Event) string { var buf bytes.Buffer - e.Stack().Frames(func(f trace.StackFrame) bool { + for f := range e.Stack().Frames() { file := strings.TrimPrefix(f.File, runtime.GOROOT()) fmt.Fprintf(&buf, "%s\n\t%s:%d\n", f.Func, file, f.Line) - return true - }) + } return buf.String() } diff --git a/src/runtime/traceallocfree.go b/src/runtime/traceallocfree.go index e6a2a79c..84188a55 100644 --- a/src/runtime/traceallocfree.go +++ b/src/runtime/traceallocfree.go @@ -8,7 +8,7 @@ package runtime import ( "internal/abi" - "runtime/internal/sys" + "internal/runtime/sys" ) // Batch type values for the alloc/free experiment. @@ -89,17 +89,17 @@ func traceSpanTypeAndClass(s *mspan) traceArg { // SpanExists records an event indicating that the span exists. func (tl traceLocker) SpanExists(s *mspan) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvSpan, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSpan, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) } // SpanAlloc records an event indicating that the span has just been allocated. func (tl traceLocker) SpanAlloc(s *mspan) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvSpanAlloc, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSpanAlloc, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) } // SpanFree records an event indicating that the span is about to be freed. func (tl traceLocker) SpanFree(s *mspan) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvSpanFree, traceSpanID(s)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSpanFree, traceSpanID(s)) } // traceSpanID creates a trace ID for the span s for the trace. @@ -111,19 +111,19 @@ func traceSpanID(s *mspan) traceArg { // The type is optional, and the size of the slot occupied the object is inferred from the // span containing it. func (tl traceLocker) HeapObjectExists(addr uintptr, typ *abi.Type) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvHeapObject, traceHeapObjectID(addr), tl.rtype(typ)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapObject, traceHeapObjectID(addr), tl.rtype(typ)) } // HeapObjectAlloc records that an object was newly allocated at addr with the provided type. // The type is optional, and the size of the slot occupied the object is inferred from the // span containing it. func (tl traceLocker) HeapObjectAlloc(addr uintptr, typ *abi.Type) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvHeapObjectAlloc, traceHeapObjectID(addr), tl.rtype(typ)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapObjectAlloc, traceHeapObjectID(addr), tl.rtype(typ)) } // HeapObjectFree records that an object at addr is about to be freed. func (tl traceLocker) HeapObjectFree(addr uintptr) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvHeapObjectFree, traceHeapObjectID(addr)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapObjectFree, traceHeapObjectID(addr)) } // traceHeapObjectID creates a trace ID for a heap object at address addr. @@ -134,18 +134,18 @@ func traceHeapObjectID(addr uintptr) traceArg { // GoroutineStackExists records that a goroutine stack already exists at address base with the provided size. func (tl traceLocker) GoroutineStackExists(base, size uintptr) { order := traceCompressStackSize(size) - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGoroutineStack, traceGoroutineStackID(base), order) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoroutineStack, traceGoroutineStackID(base), order) } // GoroutineStackAlloc records that a goroutine stack was newly allocated at address base with the provided size.. func (tl traceLocker) GoroutineStackAlloc(base, size uintptr) { order := traceCompressStackSize(size) - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGoroutineStackAlloc, traceGoroutineStackID(base), order) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoroutineStackAlloc, traceGoroutineStackID(base), order) } // GoroutineStackFree records that a goroutine stack at address base is about to be freed. func (tl traceLocker) GoroutineStackFree(base uintptr) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGoroutineStackFree, traceGoroutineStackID(base)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoroutineStackFree, traceGoroutineStackID(base)) } // traceGoroutineStackID creates a trace ID for the goroutine stack from its base address. diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 03c02f77..91c0720d 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -8,8 +8,8 @@ import ( "internal/abi" "internal/bytealg" "internal/goarch" + "internal/runtime/sys" "internal/stringslite" - "runtime/internal/sys" "unsafe" ) @@ -143,7 +143,7 @@ func (u *unwinder) initAt(pc0, sp0, lr0 uintptr, gp *g, flags unwindFlags) { // on another stack. That could confuse callers quite a bit. // Instead, we require that initAt and any other function that // accepts an sp for the current goroutine (typically obtained by - // calling getcallersp) must not run on that goroutine's stack but + // calling GetCallerSP) must not run on that goroutine's stack but // instead on the g0 stack. throw("cannot trace user goroutine on its own stack") } @@ -804,7 +804,7 @@ func traceback(pc, sp, lr uintptr, gp *g) { } // tracebacktrap is like traceback but expects that the PC and SP were obtained -// from a trap, not from gp->sched or gp->syscallpc/gp->syscallsp or getcallerpc/getcallersp. +// from a trap, not from gp->sched or gp->syscallpc/gp->syscallsp or GetCallerPC/GetCallerSP. // Because they are from a trap instead of from a saved pair, // the initial PC must not be rewound to the previous instruction. // (All the saved pairs record a PC that is a return address, so we @@ -1090,8 +1090,8 @@ func printAncestorTracebackFuncInfo(f funcInfo, pc uintptr) { // //go:linkname callers func callers(skip int, pcbuf []uintptr) int { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() gp := getg() var n int systemstack(func() { @@ -1149,11 +1149,10 @@ func showfuncinfo(sf srcFunc, firstFrame bool, calleeID abi.FuncID) bool { // It is only for runtime functions, so ASCII A-Z is fine. func isExportedRuntime(name string) bool { // Check and remove package qualifier. - n := len("runtime.") - if len(name) <= n || name[:n] != "runtime." { + name, found := stringslite.CutPrefix(name, "runtime.") + if !found { return false } - name = name[n:] rcvr := "" // Extract receiver type, if any. @@ -1239,6 +1238,9 @@ func goroutineheader(gp *g) { if gp.lockedm != 0 { print(", locked to thread") } + if sg := gp.syncGroup; sg != nil { + print(", synctest group ", sg.root.goid) + } print("]:\n") } diff --git a/src/runtime/traceback_system_test.go b/src/runtime/traceback_system_test.go index ece58e80..af20f54a 100644 --- a/src/runtime/traceback_system_test.go +++ b/src/runtime/traceback_system_test.go @@ -23,8 +23,8 @@ import ( ) // This is the entrypoint of the child process used by -// TestTracebackSystem. It prints a crash report to stdout. -func crash() { +// TestTracebackSystem/panic. It prints a crash report to stdout. +func crashViaPanic() { // Ensure that we get pc=0x%x values in the traceback. debug.SetTraceback("system") writeSentinel(os.Stdout) @@ -37,6 +37,21 @@ func crash() { select {} } +// This is the entrypoint of the child process used by +// TestTracebackSystem/trap. It prints a crash report to stdout. +func crashViaTrap() { + // Ensure that we get pc=0x%x values in the traceback. + debug.SetTraceback("system") + writeSentinel(os.Stdout) + debug.SetCrashOutput(os.Stdout, debug.CrashOptions{}) + + go func() { + // This call is typically inlined. + trap1() + }() + select {} +} + func child1() { child2() } @@ -85,6 +100,20 @@ func child7() { panic("oops") } +func trap1() { + trap2() +} + +var sinkPtr *int + +func trap2() { + trap3(sinkPtr) +} + +func trap3(i *int) { + *i = 42 +} + // TestTracebackSystem tests that the syntax of crash reports produced // by GOTRACEBACK=system (see traceback2) contains a complete, // parseable list of program counters for the running goroutine that @@ -100,46 +129,75 @@ func TestTracebackSystem(t *testing.T) { t.Skip("Can't read source code for this file on Android") } - // Fork+exec the crashing process. - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - cmd := testenv.Command(t, exe) - cmd.Env = append(cmd.Environ(), entrypointVar+"=crash") - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - cmd.Run() // expected to crash - t.Logf("stderr:\n%s\nstdout: %s\n", stderr.Bytes(), stdout.Bytes()) - crash := stdout.String() - - // If the only line is the sentinel, it wasn't a crash. - if strings.Count(crash, "\n") < 2 { - t.Fatalf("child process did not produce a crash report") - } - - // Parse the PCs out of the child's crash report. - pcs, err := parseStackPCs(crash) - if err != nil { - t.Fatal(err) - } - - // Unwind the stack using this executable's symbol table. - got := formatStack(pcs) - want := `redacted.go:0: runtime.gopanic -traceback_system_test.go:85: runtime_test.child7: panic("oops") -traceback_system_test.go:68: runtime_test.child6: child7() // appears in stack trace -traceback_system_test.go:59: runtime_test.child5: child6() // appears in stack trace -traceback_system_test.go:53: runtime_test.child4: child5() -traceback_system_test.go:49: runtime_test.child3: child4() -traceback_system_test.go:45: runtime_test.child2: child3() -traceback_system_test.go:41: runtime_test.child1: child2() -traceback_system_test.go:35: runtime_test.crash.func1: child1() + tests := []struct{ + name string + want string + }{ + { + name: "panic", + want: `redacted.go:0: runtime.gopanic +traceback_system_test.go:100: runtime_test.child7: panic("oops") +traceback_system_test.go:83: runtime_test.child6: child7() // appears in stack trace +traceback_system_test.go:74: runtime_test.child5: child6() // appears in stack trace +traceback_system_test.go:68: runtime_test.child4: child5() +traceback_system_test.go:64: runtime_test.child3: child4() +traceback_system_test.go:60: runtime_test.child2: child3() +traceback_system_test.go:56: runtime_test.child1: child2() +traceback_system_test.go:35: runtime_test.crashViaPanic.func1: child1() redacted.go:0: runtime.goexit -` - if strings.TrimSpace(got) != strings.TrimSpace(want) { - t.Errorf("got:\n%swant:\n%s", got, want) +`, + }, + { + // Test panic via trap. x/telemetry is aware that trap + // PCs follow runtime.sigpanic and need to be + // incremented to offset the decrement done by + // CallersFrames. + name: "trap", + want: `redacted.go:0: runtime.gopanic +redacted.go:0: runtime.panicmem +redacted.go:0: runtime.sigpanic +traceback_system_test.go:114: runtime_test.trap3: *i = 42 +traceback_system_test.go:110: runtime_test.trap2: trap3(sinkPtr) +traceback_system_test.go:104: runtime_test.trap1: trap2() +traceback_system_test.go:50: runtime_test.crashViaTrap.func1: trap1() +redacted.go:0: runtime.goexit +`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Fork+exec the crashing process. + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := testenv.Command(t, exe) + cmd.Env = append(cmd.Environ(), entrypointVar+"="+tc.name) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.Run() // expected to crash + t.Logf("stderr:\n%s\nstdout: %s\n", stderr.Bytes(), stdout.Bytes()) + crash := stdout.String() + + // If the only line is the sentinel, it wasn't a crash. + if strings.Count(crash, "\n") < 2 { + t.Fatalf("child process did not produce a crash report") + } + + // Parse the PCs out of the child's crash report. + pcs, err := parseStackPCs(crash) + if err != nil { + t.Fatal(err) + } + + // Unwind the stack using this executable's symbol table. + got := formatStack(pcs) + if strings.TrimSpace(got) != strings.TrimSpace(tc.want) { + t.Errorf("got:\n%swant:\n%s", got, tc.want) + } + }) } } @@ -154,6 +212,35 @@ redacted.go:0: runtime.goexit // // (Copied from golang.org/x/telemetry/crashmonitor.parseStackPCs.) func parseStackPCs(crash string) ([]uintptr, error) { + // getSymbol parses the symbol name out of a line of the form: + // SYMBOL(ARGS) + // + // Note: SYMBOL may contain parens "pkg.(*T).method". However, type + // parameters are always replaced with ..., so they cannot introduce + // more parens. e.g., "pkg.(*T[...]).method". + // + // ARGS can contain parens. We want the first paren that is not + // immediately preceded by a ".". + // + // TODO(prattmic): This is mildly complicated and is only used to find + // runtime.sigpanic, so perhaps simplify this by checking explicitly + // for sigpanic. + getSymbol := func(line string) (string, error) { + var prev rune + for i, c := range line { + if line[i] != '(' { + prev = c + continue + } + if prev == '.' { + prev = c + continue + } + return line[:i], nil + } + return "", fmt.Errorf("no symbol for stack frame: %s", line) + } + // getPC parses the PC out of a line of the form: // \tFILE:LINE +0xRELPC sp=... fp=... pc=... getPC := func(line string) (uint64, error) { @@ -170,6 +257,9 @@ func parseStackPCs(crash string) ([]uintptr, error) { childSentinel = sentinel() on = false // are we in the first running goroutine? lines = strings.Split(crash, "\n") + symLine = true // within a goroutine, every other line is a symbol or file/line/pc location, starting with symbol. + currSymbol string + prevSymbol string // symbol of the most recent previous frame with a PC. ) for i := 0; i < len(lines); i++ { line := lines[i] @@ -212,21 +302,76 @@ func parseStackPCs(crash string) ([]uintptr, error) { // Note: SYMBOL may contain parens "pkg.(*T).method" // The RELPC is sometimes missing. - // Skip the symbol(args) line. - i++ - if i == len(lines) { - break - } - line = lines[i] + if symLine { + var err error + currSymbol, err = getSymbol(line) + if err != nil { + return nil, fmt.Errorf("error extracting symbol: %v", err) + } - // Parse the PC, and correct for the parent and child's - // different mappings of the text section. - pc, err := getPC(line) - if err != nil { - // Inlined frame, perhaps; skip it. - continue + symLine = false // Next line is FILE:LINE. + } else { + // Parse the PC, and correct for the parent and child's + // different mappings of the text section. + pc, err := getPC(line) + if err != nil { + // Inlined frame, perhaps; skip it. + + // Done with this frame. Next line is a new frame. + // + // Don't update prevSymbol; we only want to + // track frames with a PC. + currSymbol = "" + symLine = true + continue + } + + pc = pc-parentSentinel+childSentinel + + // If the previous frame was sigpanic, then this frame + // was a trap (e.g., SIGSEGV). + // + // Typically all middle frames are calls, and report + // the "return PC". That is, the instruction following + // the CALL where the callee will eventually return to. + // + // runtime.CallersFrames is aware of this property and + // will decrement each PC by 1 to "back up" to the + // location of the CALL, which is the actual line + // number the user expects. + // + // This does not work for traps, as a trap is not a + // call, so the reported PC is not the return PC, but + // the actual PC of the trap. + // + // runtime.Callers is aware of this and will + // intentionally increment trap PCs in order to correct + // for the decrement performed by + // runtime.CallersFrames. See runtime.tracebackPCs and + // runtume.(*unwinder).symPC. + // + // We must emulate the same behavior, otherwise we will + // report the location of the instruction immediately + // prior to the trap, which may be on a different line, + // or even a different inlined functions. + // + // TODO(prattmic): The runtime applies the same trap + // behavior for other "injected calls", see injectCall + // in runtime.(*unwinder).next. Do we want to handle + // those as well? I don't believe we'd ever see + // runtime.asyncPreempt or runtime.debugCallV2 in a + // typical crash. + if prevSymbol == "runtime.sigpanic" { + pc++ + } + + pcs = append(pcs, uintptr(pc)) + + // Done with this frame. Next line is a new frame. + prevSymbol = currSymbol + currSymbol = "" + symLine = true } - pcs = append(pcs, uintptr(pc-parentSentinel+childSentinel)) } return pcs, nil } diff --git a/src/runtime/tracebuf.go b/src/runtime/tracebuf.go index db4adf53..0849a578 100644 --- a/src/runtime/tracebuf.go +++ b/src/runtime/tracebuf.go @@ -7,7 +7,7 @@ package runtime import ( - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) @@ -24,12 +24,31 @@ const traceBytesPerNumber = 10 // we can change it if it's deemed too error-prone. type traceWriter struct { traceLocker + exp traceExperiment *traceBuf } -// write returns an a traceWriter that writes into the current M's stream. +// writer returns an a traceWriter that writes into the current M's stream. +// +// Once this is called, the caller must guard against stack growth until +// end is called on it. Therefore, it's highly recommended to use this +// API in a "fluent" style, for example tl.writer().event(...).end(). +// Better yet, callers just looking to write events should use eventWriter +// when possible, which is a much safer wrapper around this function. +// +// nosplit to allow for safe reentrant tracing from stack growth paths. +// +//go:nosplit func (tl traceLocker) writer() traceWriter { - return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2]} + if debugTraceReentrancy { + // Checks that the invariants of this function are being upheld. + gp := getg() + if gp == gp.m.curg { + tl.mp.trace.oldthrowsplit = gp.throwsplit + gp.throwsplit = true + } + } + return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2][traceNoExperiment]} } // unsafeTraceWriter produces a traceWriter that doesn't lock the trace. @@ -38,33 +57,88 @@ func (tl traceLocker) writer() traceWriter { // - Another traceLocker is held. // - trace.gen is prevented from advancing. // +// This does not have the same stack growth restrictions as traceLocker.writer. +// // buf may be nil. func unsafeTraceWriter(gen uintptr, buf *traceBuf) traceWriter { return traceWriter{traceLocker: traceLocker{gen: gen}, traceBuf: buf} } +// event writes out the bytes of an event into the event stream. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit +func (w traceWriter) event(ev traceEv, args ...traceArg) traceWriter { + // N.B. Everything in this call must be nosplit to maintain + // the stack growth related invariants for writing events. + + // Make sure we have room. + w, _ = w.ensure(1 + (len(args)+1)*traceBytesPerNumber) + + // Compute the timestamp diff that we'll put in the trace. + ts := traceClockNow() + if ts <= w.traceBuf.lastTime { + ts = w.traceBuf.lastTime + 1 + } + tsDiff := uint64(ts - w.traceBuf.lastTime) + w.traceBuf.lastTime = ts + + // Write out event. + w.byte(byte(ev)) + w.varint(tsDiff) + for _, arg := range args { + w.varint(uint64(arg)) + } + return w +} + // end writes the buffer back into the m. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (w traceWriter) end() { if w.mp == nil { // Tolerate a nil mp. It makes code that creates traceWriters directly // less error-prone. return } - w.mp.trace.buf[w.gen%2] = w.traceBuf + w.mp.trace.buf[w.gen%2][w.exp] = w.traceBuf + if debugTraceReentrancy { + // The writer is no longer live, we can drop throwsplit (if it wasn't + // already set upon entry). + gp := getg() + if gp == gp.m.curg { + gp.throwsplit = w.mp.trace.oldthrowsplit + } + } } // ensure makes sure that at least maxSize bytes are available to write. // // Returns whether the buffer was flushed. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (w traceWriter) ensure(maxSize int) (traceWriter, bool) { refill := w.traceBuf == nil || !w.available(maxSize) if refill { - w = w.refill(traceNoExperiment) + w = w.refill() } return w, refill } // flush puts w.traceBuf on the queue of full buffers. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (w traceWriter) flush() traceWriter { systemstack(func() { lock(&trace.lock) @@ -78,9 +152,7 @@ func (w traceWriter) flush() traceWriter { } // refill puts w.traceBuf on the queue of full buffers and refresh's w's buffer. -// -// exp indicates whether the refilled batch should be EvExperimentalBatch. -func (w traceWriter) refill(exp traceExperiment) traceWriter { +func (w traceWriter) refill() traceWriter { systemstack(func() { lock(&trace.lock) if w.traceBuf != nil { @@ -114,11 +186,11 @@ func (w traceWriter) refill(exp traceExperiment) traceWriter { } // Write the buffer's header. - if exp == traceNoExperiment { + if w.exp == traceNoExperiment { w.byte(byte(traceEvEventBatch)) } else { w.byte(byte(traceEvExperimentalBatch)) - w.byte(byte(exp)) + w.byte(byte(w.exp)) } w.varint(uint64(w.gen)) w.varint(uint64(mID)) @@ -179,12 +251,22 @@ type traceBuf struct { } // byte appends v to buf. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (buf *traceBuf) byte(v byte) { buf.arr[buf.pos] = v buf.pos++ } // varint appends v to buf in little-endian-base-128 encoding. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (buf *traceBuf) varint(v uint64) { pos := buf.pos arr := buf.arr[pos : pos+traceBytesPerNumber] @@ -203,6 +285,11 @@ func (buf *traceBuf) varint(v uint64) { // varintReserve reserves enough space in buf to hold any varint. // // Space reserved this way can be filled in with the varintAt method. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (buf *traceBuf) varintReserve() int { p := buf.pos buf.pos += traceBytesPerNumber @@ -210,10 +297,19 @@ func (buf *traceBuf) varintReserve() int { } // stringData appends s's data directly to buf. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (buf *traceBuf) stringData(s string) { buf.pos += copy(buf.arr[buf.pos:], s) } +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (buf *traceBuf) available(size int) bool { return len(buf.arr)-buf.pos >= size } @@ -222,6 +318,11 @@ func (buf *traceBuf) available(size int) bool { // consumes traceBytesPerNumber bytes. This is intended for when the caller // needs to reserve space for a varint but can't populate it until later. // Use varintReserve to reserve this space. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (buf *traceBuf) varintAt(pos int, v uint64) { for i := 0; i < traceBytesPerNumber; i++ { if i < traceBytesPerNumber-1 { diff --git a/src/runtime/traceevent.go b/src/runtime/traceevent.go index 2a869fb5..51d23688 100644 --- a/src/runtime/traceevent.go +++ b/src/runtime/traceevent.go @@ -8,7 +8,7 @@ package runtime import ( "internal/abi" - "runtime/internal/sys" + "internal/runtime/sys" ) // Event types in the trace, args are given in square brackets. @@ -102,7 +102,7 @@ type traceArg uint64 // See the comment on traceWriter about style for more details as to why // this type and its methods are structured the way they are. type traceEventWriter struct { - w traceWriter + tl traceLocker } // eventWriter creates a new traceEventWriter. It is the main entrypoint for writing trace events. @@ -119,54 +119,18 @@ type traceEventWriter struct { // // In this case, the default status should be traceGoBad or traceProcBad to help identify bugs sooner. func (tl traceLocker) eventWriter(goStatus traceGoStatus, procStatus traceProcStatus) traceEventWriter { - w := tl.writer() if pp := tl.mp.p.ptr(); pp != nil && !pp.trace.statusWasTraced(tl.gen) && pp.trace.acquireStatus(tl.gen) { - w = w.writeProcStatus(uint64(pp.id), procStatus, pp.trace.inSweep) + tl.writer().writeProcStatus(uint64(pp.id), procStatus, pp.trace.inSweep).end() } if gp := tl.mp.curg; gp != nil && !gp.trace.statusWasTraced(tl.gen) && gp.trace.acquireStatus(tl.gen) { - w = w.writeGoStatus(uint64(gp.goid), int64(tl.mp.procid), goStatus, gp.inMarkAssist, 0 /* no stack */) + tl.writer().writeGoStatus(uint64(gp.goid), int64(tl.mp.procid), goStatus, gp.inMarkAssist, 0 /* no stack */).end() } - return traceEventWriter{w} + return traceEventWriter{tl} } -// commit writes out a trace event and calls end. It's a helper to make the -// common case of writing out a single event less error-prone. -func (e traceEventWriter) commit(ev traceEv, args ...traceArg) { - e = e.write(ev, args...) - e.end() -} - -// write writes an event into the trace. -func (e traceEventWriter) write(ev traceEv, args ...traceArg) traceEventWriter { - e.w = e.w.event(ev, args...) - return e -} - -// end finishes writing to the trace. The traceEventWriter must not be used after this call. -func (e traceEventWriter) end() { - e.w.end() -} - -// traceEventWrite is the part of traceEvent that actually writes the event. -func (w traceWriter) event(ev traceEv, args ...traceArg) traceWriter { - // Make sure we have room. - w, _ = w.ensure(1 + (len(args)+1)*traceBytesPerNumber) - - // Compute the timestamp diff that we'll put in the trace. - ts := traceClockNow() - if ts <= w.traceBuf.lastTime { - ts = w.traceBuf.lastTime + 1 - } - tsDiff := uint64(ts - w.traceBuf.lastTime) - w.traceBuf.lastTime = ts - - // Write out event. - w.byte(byte(ev)) - w.varint(tsDiff) - for _, arg := range args { - w.varint(uint64(arg)) - } - return w +// event writes out a trace event. +func (e traceEventWriter) event(ev traceEv, args ...traceArg) { + e.tl.writer().event(ev, args...).end() } // stack takes a stack trace skipping the provided number of frames. diff --git a/src/runtime/traceexp.go b/src/runtime/traceexp.go index 9fc85df5..13eec0c0 100644 --- a/src/runtime/traceexp.go +++ b/src/runtime/traceexp.go @@ -4,34 +4,25 @@ package runtime -// traceExpWriter is a wrapper around trace writer that produces traceEvExperimentalBatch -// batches. This means that the data written to the writer need not conform to the standard -// trace format. -type traceExpWriter struct { - traceWriter - exp traceExperiment +// expWriter returns a traceWriter that writes into the current M's stream for +// the given experiment. +func (tl traceLocker) expWriter(exp traceExperiment) traceWriter { + return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2][exp], exp: exp} } -// unsafeTraceExpWriter produces a traceExpWriter that doesn't lock the trace. +// unsafeTraceExpWriter produces a traceWriter for experimental trace batches +// that doesn't lock the trace. Data written to experimental batches need not +// conform to the standard trace format. // // It should only be used in contexts where either: // - Another traceLocker is held. // - trace.gen is prevented from advancing. // -// buf may be nil. -func unsafeTraceExpWriter(gen uintptr, buf *traceBuf, exp traceExperiment) traceExpWriter { - return traceExpWriter{traceWriter{traceLocker: traceLocker{gen: gen}, traceBuf: buf}, exp} -} - -// ensure makes sure that at least maxSize bytes are available to write. +// This does not have the same stack growth restrictions as traceLocker.writer. // -// Returns whether the buffer was flushed. -func (w traceExpWriter) ensure(maxSize int) (traceExpWriter, bool) { - refill := w.traceBuf == nil || !w.available(maxSize) - if refill { - w.traceWriter = w.traceWriter.refill(w.exp) - } - return w, refill +// buf may be nil. +func unsafeTraceExpWriter(gen uintptr, buf *traceBuf, exp traceExperiment) traceWriter { + return traceWriter{traceLocker: traceLocker{gen: gen}, traceBuf: buf, exp: exp} } // traceExperiment is an enumeration of the different kinds of experiments supported for tracing. @@ -43,6 +34,10 @@ const ( // traceExperimentAllocFree is an experiment to add alloc/free events to the trace. traceExperimentAllocFree + + // traceNumExperiments is the number of trace experiments (and 1 higher than + // the highest numbered experiment). + traceNumExperiments ) // Experimental events. diff --git a/src/runtime/tracemap.go b/src/runtime/tracemap.go index 5b2718c8..9efa325c 100644 --- a/src/runtime/tracemap.go +++ b/src/runtime/tracemap.go @@ -19,7 +19,7 @@ import ( "internal/cpu" "internal/goarch" "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/traceregion.go b/src/runtime/traceregion.go index 43eef9c9..b45093ec 100644 --- a/src/runtime/traceregion.go +++ b/src/runtime/traceregion.go @@ -8,7 +8,7 @@ package runtime import ( "internal/runtime/atomic" - "runtime/internal/sys" + "internal/runtime/sys" "unsafe" ) diff --git a/src/runtime/traceruntime.go b/src/runtime/traceruntime.go index 7c4cb550..284e6130 100644 --- a/src/runtime/traceruntime.go +++ b/src/runtime/traceruntime.go @@ -24,9 +24,11 @@ func (s *gTraceState) reset() { // mTraceState is per-M state for the tracer. type mTraceState struct { - seqlock atomic.Uintptr // seqlock indicating that this M is writing to a trace buffer. - buf [2]*traceBuf // Per-M traceBuf for writing. Indexed by trace.gen%2. - link *m // Snapshot of alllink or freelink. + seqlock atomic.Uintptr // seqlock indicating that this M is writing to a trace buffer. + buf [2][traceNumExperiments]*traceBuf // Per-M traceBuf for writing. Indexed by trace.gen%2. + link *m // Snapshot of alllink or freelink. + reentered uint32 // Whether we've reentered tracing from within tracing. + oldthrowsplit bool // gp.throwsplit upon calling traceLocker.writer. For debugging. } // pTraceState is per-P state for the tracer. @@ -100,6 +102,7 @@ const ( traceBlockUntilGCEnds traceBlockSleep traceBlockGCWeakToStrongWait + traceBlockSynctest ) var traceBlockReasonStrings = [...]string{ @@ -119,6 +122,7 @@ var traceBlockReasonStrings = [...]string{ traceBlockUntilGCEnds: "wait until GC ends", traceBlockSleep: "sleep", traceBlockGCWeakToStrongWait: "GC weak to strong wait", + traceBlockSynctest: "synctest", } // traceGoStopReason is an enumeration of reasons a goroutine might yield. @@ -186,22 +190,6 @@ func traceAcquire() traceLocker { return traceAcquireEnabled() } -// traceTryAcquire is like traceAcquire, but may return an invalid traceLocker even -// if tracing is enabled. For example, it will return !ok if traceAcquire is being -// called with an active traceAcquire on the M (reentrant locking). This exists for -// optimistically emitting events in the few contexts where tracing is now allowed. -// -// nosplit for alignment with traceTryAcquire, so it can be used in the -// same contexts. -// -//go:nosplit -func traceTryAcquire() traceLocker { - if !traceEnabled() { - return traceLocker{} - } - return traceTryAcquireEnabled() -} - // traceAcquireEnabled is the traceEnabled path for traceAcquire. It's explicitly // broken out to make traceAcquire inlineable to keep the overhead of the tracer // when it's disabled low. @@ -218,6 +206,14 @@ func traceAcquireEnabled() traceLocker { // Prevent preemption. mp := acquirem() + // Check if we're already tracing. It's safe to be reentrant in general, + // because this function (and the invariants of traceLocker.writer) ensure + // that it is. + if mp.trace.seqlock.Load()%2 == 1 { + mp.trace.reentered++ + return traceLocker{mp, trace.gen.Load()} + } + // Acquire the trace seqlock. This prevents traceAdvance from moving forward // until all Ms are observed to be outside of their seqlock critical section. // @@ -226,7 +222,7 @@ func traceAcquireEnabled() traceLocker { // doing. seq := mp.trace.seqlock.Add(1) if debugTraceReentrancy && seq%2 != 1 { - throw("bad use of trace.seqlock or tracer is reentrant") + throw("bad use of trace.seqlock") } // N.B. This load of gen appears redundant with the one in traceEnabled. @@ -246,26 +242,6 @@ func traceAcquireEnabled() traceLocker { return traceLocker{mp, gen} } -// traceTryAcquireEnabled is like traceAcquireEnabled but may return an invalid -// traceLocker under some conditions. See traceTryAcquire for more details. -// -// nosplit for alignment with traceAcquireEnabled, so it can be used in the -// same contexts. -// -//go:nosplit -func traceTryAcquireEnabled() traceLocker { - // Any time we acquire a traceLocker, we may flush a trace buffer. But - // buffer flushes are rare. Record the lock edge even if it doesn't happen - // this time. - lockRankMayTraceFlush() - - // Check if we're already locked. If so, return an invalid traceLocker. - if getg().m.trace.seqlock.Load()%2 == 1 { - return traceLocker{} - } - return traceAcquireEnabled() -} - // ok returns true if the traceLocker is valid (i.e. tracing is enabled). // // nosplit because it's called on the syscall path when stack movement is forbidden. @@ -281,10 +257,14 @@ func (tl traceLocker) ok() bool { // //go:nosplit func traceRelease(tl traceLocker) { - seq := tl.mp.trace.seqlock.Add(1) - if debugTraceReentrancy && seq%2 != 0 { - print("runtime: seq=", seq, "\n") - throw("bad use of trace.seqlock") + if tl.mp.trace.reentered > 0 { + tl.mp.trace.reentered-- + } else { + seq := tl.mp.trace.seqlock.Add(1) + if debugTraceReentrancy && seq%2 != 0 { + print("runtime: seq=", seq, "\n") + throw("bad use of trace.seqlock") + } } releasem(tl.mp) } @@ -303,7 +283,7 @@ func traceExitedSyscall() { // Gomaxprocs emits a ProcsChange event. func (tl traceLocker) Gomaxprocs(procs int32) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvProcsChange, traceArg(procs), tl.stack(1)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvProcsChange, traceArg(procs), tl.stack(1)) } // ProcStart traces a ProcStart event. @@ -314,14 +294,14 @@ func (tl traceLocker) ProcStart() { // Procs are typically started within the scheduler when there is no user goroutine. If there is a user goroutine, // it must be in _Gsyscall because the only time a goroutine is allowed to have its Proc moved around from under it // is during a syscall. - tl.eventWriter(traceGoSyscall, traceProcIdle).commit(traceEvProcStart, traceArg(pp.id), pp.trace.nextSeq(tl.gen)) + tl.eventWriter(traceGoSyscall, traceProcIdle).event(traceEvProcStart, traceArg(pp.id), pp.trace.nextSeq(tl.gen)) } // ProcStop traces a ProcStop event. func (tl traceLocker) ProcStop(pp *p) { // The only time a goroutine is allowed to have its Proc moved around // from under it is during a syscall. - tl.eventWriter(traceGoSyscall, traceProcRunning).commit(traceEvProcStop) + tl.eventWriter(traceGoSyscall, traceProcRunning).event(traceEvProcStop) } // GCActive traces a GCActive event. @@ -329,7 +309,7 @@ func (tl traceLocker) ProcStop(pp *p) { // Must be emitted by an actively running goroutine on an active P. This restriction can be changed // easily and only depends on where it's currently called. func (tl traceLocker) GCActive() { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGCActive, traceArg(trace.seqGC)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCActive, traceArg(trace.seqGC)) // N.B. Only one GC can be running at a time, so this is naturally // serialized by the caller. trace.seqGC++ @@ -340,7 +320,7 @@ func (tl traceLocker) GCActive() { // Must be emitted by an actively running goroutine on an active P. This restriction can be changed // easily and only depends on where it's currently called. func (tl traceLocker) GCStart() { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGCBegin, traceArg(trace.seqGC), tl.stack(3)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCBegin, traceArg(trace.seqGC), tl.stack(3)) // N.B. Only one GC can be running at a time, so this is naturally // serialized by the caller. trace.seqGC++ @@ -351,7 +331,7 @@ func (tl traceLocker) GCStart() { // Must be emitted by an actively running goroutine on an active P. This restriction can be changed // easily and only depends on where it's currently called. func (tl traceLocker) GCDone() { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGCEnd, traceArg(trace.seqGC)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCEnd, traceArg(trace.seqGC)) // N.B. Only one GC can be running at a time, so this is naturally // serialized by the caller. trace.seqGC++ @@ -361,14 +341,14 @@ func (tl traceLocker) GCDone() { func (tl traceLocker) STWStart(reason stwReason) { // Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the // runtime's state tracking, but it's more accurate and doesn't result in any loss of information. - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvSTWBegin, tl.string(reason.String()), tl.stack(2)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSTWBegin, tl.string(reason.String()), tl.stack(2)) } // STWDone traces a STWEnd event. func (tl traceLocker) STWDone() { // Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the // runtime's state tracking, but it's more accurate and doesn't result in any loss of information. - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvSTWEnd) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSTWEnd) } // GCSweepStart prepares to trace a sweep loop. This does not @@ -400,7 +380,7 @@ func (tl traceLocker) GCSweepSpan(bytesSwept uintptr) { pp := tl.mp.p.ptr() if pp.trace.maySweep { if pp.trace.swept == 0 { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGCSweepBegin, tl.stack(1)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCSweepBegin, tl.stack(1)) pp.trace.inSweep = true } pp.trace.swept += bytesSwept @@ -418,7 +398,7 @@ func (tl traceLocker) GCSweepDone() { throw("missing traceGCSweepStart") } if pp.trace.inSweep { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGCSweepEnd, traceArg(pp.trace.swept), traceArg(pp.trace.reclaimed)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCSweepEnd, traceArg(pp.trace.swept), traceArg(pp.trace.reclaimed)) pp.trace.inSweep = false } pp.trace.maySweep = false @@ -426,12 +406,12 @@ func (tl traceLocker) GCSweepDone() { // GCMarkAssistStart emits a MarkAssistBegin event. func (tl traceLocker) GCMarkAssistStart() { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGCMarkAssistBegin, tl.stack(1)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCMarkAssistBegin, tl.stack(1)) } // GCMarkAssistDone emits a MarkAssistEnd event. func (tl traceLocker) GCMarkAssistDone() { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGCMarkAssistEnd) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCMarkAssistEnd) } // GoCreate emits a GoCreate event. @@ -441,7 +421,7 @@ func (tl traceLocker) GoCreate(newg *g, pc uintptr, blocked bool) { if blocked { ev = traceEvGoCreateBlocked } - tl.eventWriter(traceGoRunning, traceProcRunning).commit(ev, traceArg(newg.goid), tl.startPC(pc), tl.stack(2)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(ev, traceArg(newg.goid), tl.startPC(pc), tl.stack(2)) } // GoStart emits a GoStart event. @@ -451,18 +431,17 @@ func (tl traceLocker) GoStart() { gp := getg().m.curg pp := gp.m.p w := tl.eventWriter(traceGoRunnable, traceProcRunning) - w = w.write(traceEvGoStart, traceArg(gp.goid), gp.trace.nextSeq(tl.gen)) + w.event(traceEvGoStart, traceArg(gp.goid), gp.trace.nextSeq(tl.gen)) if pp.ptr().gcMarkWorkerMode != gcMarkWorkerNotWorker { - w = w.write(traceEvGoLabel, trace.markWorkerLabels[tl.gen%2][pp.ptr().gcMarkWorkerMode]) + w.event(traceEvGoLabel, trace.markWorkerLabels[tl.gen%2][pp.ptr().gcMarkWorkerMode]) } - w.end() } // GoEnd emits a GoDestroy event. // // TODO(mknyszek): Rename this to GoDestroy. func (tl traceLocker) GoEnd() { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGoDestroy) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoDestroy) } // GoSched emits a GoStop event with a GoSched reason. @@ -477,7 +456,7 @@ func (tl traceLocker) GoPreempt() { // GoStop emits a GoStop event with the provided reason. func (tl traceLocker) GoStop(reason traceGoStopReason) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGoStop, traceArg(trace.goStopReasons[tl.gen%2][reason]), tl.stack(1)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoStop, traceArg(trace.goStopReasons[tl.gen%2][reason]), tl.stack(1)) } // GoPark emits a GoBlock event with the provided reason. @@ -485,44 +464,38 @@ func (tl traceLocker) GoStop(reason traceGoStopReason) { // TODO(mknyszek): Replace traceBlockReason with waitReason. It's silly // that we have both, and waitReason is way more descriptive. func (tl traceLocker) GoPark(reason traceBlockReason, skip int) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGoBlock, traceArg(trace.goBlockReasons[tl.gen%2][reason]), tl.stack(skip)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoBlock, traceArg(trace.goBlockReasons[tl.gen%2][reason]), tl.stack(skip)) } // GoUnpark emits a GoUnblock event. func (tl traceLocker) GoUnpark(gp *g, skip int) { // Emit a GoWaiting status if necessary for the unblocked goroutine. - w := tl.eventWriter(traceGoRunning, traceProcRunning) - // Careful: don't use the event writer. We never want status or in-progress events - // to trigger more in-progress events. - w.w = emitUnblockStatus(w.w, gp, tl.gen) - w.commit(traceEvGoUnblock, traceArg(gp.goid), gp.trace.nextSeq(tl.gen), tl.stack(skip)) + tl.emitUnblockStatus(gp, tl.gen) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoUnblock, traceArg(gp.goid), gp.trace.nextSeq(tl.gen), tl.stack(skip)) } -// GoCoroswitch emits a GoSwitch event. If destroy is true, the calling goroutine +// GoSwitch emits a GoSwitch event. If destroy is true, the calling goroutine // is simultaneously being destroyed. func (tl traceLocker) GoSwitch(nextg *g, destroy bool) { // Emit a GoWaiting status if necessary for the unblocked goroutine. + tl.emitUnblockStatus(nextg, tl.gen) w := tl.eventWriter(traceGoRunning, traceProcRunning) - // Careful: don't use the event writer. We never want status or in-progress events - // to trigger more in-progress events. - w.w = emitUnblockStatus(w.w, nextg, tl.gen) ev := traceEvGoSwitch if destroy { ev = traceEvGoSwitchDestroy } - w.commit(ev, traceArg(nextg.goid), nextg.trace.nextSeq(tl.gen)) + w.event(ev, traceArg(nextg.goid), nextg.trace.nextSeq(tl.gen)) } // emitUnblockStatus emits a GoStatus GoWaiting event for a goroutine about to be // unblocked to the trace writer. -func emitUnblockStatus(w traceWriter, gp *g, gen uintptr) traceWriter { +func (tl traceLocker) emitUnblockStatus(gp *g, gen uintptr) { if !gp.trace.statusWasTraced(gen) && gp.trace.acquireStatus(gen) { // TODO(go.dev/issue/65634): Although it would be nice to add a stack trace here of gp, // we cannot safely do so. gp is in _Gwaiting and so we don't have ownership of its stack. // We can fix this by acquiring the goroutine's scan bit. - w = w.writeGoStatus(gp.goid, -1, traceGoWaiting, gp.inMarkAssist, 0) + tl.writer().writeGoStatus(gp.goid, -1, traceGoWaiting, gp.inMarkAssist, 0).end() } - return w } // GoSysCall emits a GoSyscallBegin event. @@ -532,7 +505,7 @@ func (tl traceLocker) GoSysCall() { // Scribble down the M that the P is currently attached to. pp := tl.mp.p.ptr() pp.trace.mSyscallID = int64(tl.mp.procid) - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvGoSyscallBegin, pp.trace.nextSeq(tl.gen), tl.stack(1)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoSyscallBegin, pp.trace.nextSeq(tl.gen), tl.stack(1)) } // GoSysExit emits a GoSyscallEnd event, possibly along with a GoSyscallBlocked event @@ -553,7 +526,7 @@ func (tl traceLocker) GoSysExit(lostP bool) { } else { tl.mp.p.ptr().trace.mSyscallID = -1 } - tl.eventWriter(traceGoSyscall, procStatus).commit(ev) + tl.eventWriter(traceGoSyscall, procStatus).event(ev) } // ProcSteal indicates that our current M stole a P from another M. @@ -566,6 +539,17 @@ func (tl traceLocker) ProcSteal(pp *p, inSyscall bool) { mStolenFrom := pp.trace.mSyscallID pp.trace.mSyscallID = -1 + // Emit the status of the P we're stealing. We may be just about to do this when creating the event + // writer but it's not guaranteed, even if inSyscall is true. Although it might seem like from a + // syscall context we're always stealing a P for ourselves, we may have not wired it up yet (so + // it wouldn't be visible to eventWriter) or we may not even intend to wire it up to ourselves + // at all (e.g. entersyscall_gcwait). + if !pp.trace.statusWasTraced(tl.gen) && pp.trace.acquireStatus(tl.gen) { + // Careful: don't use the event writer. We never want status or in-progress events + // to trigger more in-progress events. + tl.writer().writeProcStatus(uint64(pp.id), traceProcSyscallAbandoned, pp.trace.inSweep).end() + } + // The status of the proc and goroutine, if we need to emit one here, is not evident from the // context of just emitting this event alone. There are two cases. Either we're trying to steal // the P just to get its attention (e.g. STW or sysmon retake) or we're trying to steal a P for @@ -578,24 +562,12 @@ func (tl traceLocker) ProcSteal(pp *p, inSyscall bool) { goStatus = traceGoSyscall procStatus = traceProcSyscallAbandoned } - w := tl.eventWriter(goStatus, procStatus) - - // Emit the status of the P we're stealing. We may have *just* done this when creating the event - // writer but it's not guaranteed, even if inSyscall is true. Although it might seem like from a - // syscall context we're always stealing a P for ourselves, we may have not wired it up yet (so - // it wouldn't be visible to eventWriter) or we may not even intend to wire it up to ourselves - // at all (e.g. entersyscall_gcwait). - if !pp.trace.statusWasTraced(tl.gen) && pp.trace.acquireStatus(tl.gen) { - // Careful: don't use the event writer. We never want status or in-progress events - // to trigger more in-progress events. - w.w = w.w.writeProcStatus(uint64(pp.id), traceProcSyscallAbandoned, pp.trace.inSweep) - } - w.commit(traceEvProcSteal, traceArg(pp.id), pp.trace.nextSeq(tl.gen), traceArg(mStolenFrom)) + tl.eventWriter(goStatus, procStatus).event(traceEvProcSteal, traceArg(pp.id), pp.trace.nextSeq(tl.gen), traceArg(mStolenFrom)) } // HeapAlloc emits a HeapAlloc event. func (tl traceLocker) HeapAlloc(live uint64) { - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvHeapAlloc, traceArg(live)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapAlloc, traceArg(live)) } // HeapGoal reads the current heap goal and emits a HeapGoal event. @@ -605,7 +577,7 @@ func (tl traceLocker) HeapGoal() { // Heap-based triggering is disabled. heapGoal = 0 } - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvHeapGoal, traceArg(heapGoal)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapGoal, traceArg(heapGoal)) } // GoCreateSyscall indicates that a goroutine has transitioned from dead to GoSyscall. @@ -618,7 +590,7 @@ func (tl traceLocker) GoCreateSyscall(gp *g) { // N.B. We should never trace a status for this goroutine (which we're currently running on), // since we want this to appear like goroutine creation. gp.trace.setStatusTraced(tl.gen) - tl.eventWriter(traceGoBad, traceProcBad).commit(traceEvGoCreateSyscall, traceArg(gp.goid)) + tl.eventWriter(traceGoBad, traceProcBad).event(traceEvGoCreateSyscall, traceArg(gp.goid)) } // GoDestroySyscall indicates that a goroutine has transitioned from GoSyscall to dead. @@ -630,7 +602,7 @@ func (tl traceLocker) GoCreateSyscall(gp *g) { func (tl traceLocker) GoDestroySyscall() { // N.B. If we trace a status here, we must never have a P, and we must be on a goroutine // that is in the syscall state. - tl.eventWriter(traceGoSyscall, traceProcBad).commit(traceEvGoDestroySyscall) + tl.eventWriter(traceGoSyscall, traceProcBad).event(traceEvGoDestroySyscall) } // To access runtime functions from runtime/trace. @@ -645,7 +617,7 @@ func trace_userTaskCreate(id, parentID uint64, taskType string) { // Need to do this check because the caller won't have it. return } - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvUserTaskBegin, traceArg(id), traceArg(parentID), tl.string(taskType), tl.stack(3)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvUserTaskBegin, traceArg(id), traceArg(parentID), tl.string(taskType), tl.stack(3)) traceRelease(tl) } @@ -658,11 +630,11 @@ func trace_userTaskEnd(id uint64) { // Need to do this check because the caller won't have it. return } - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvUserTaskEnd, traceArg(id), tl.stack(2)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvUserTaskEnd, traceArg(id), tl.stack(2)) traceRelease(tl) } -// trace_userTaskEnd emits a UserRegionBegin or UserRegionEnd event, +// trace_userRegion emits a UserRegionBegin or UserRegionEnd event, // depending on mode (0 == Begin, 1 == End). // // TODO(mknyszek): Just make this two functions. @@ -683,11 +655,11 @@ func trace_userRegion(id, mode uint64, name string) { default: return } - tl.eventWriter(traceGoRunning, traceProcRunning).commit(ev, traceArg(id), tl.string(name), tl.stack(3)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(ev, traceArg(id), tl.string(name), tl.stack(3)) traceRelease(tl) } -// trace_userTaskEnd emits a UserRegionBegin or UserRegionEnd event. +// trace_userLog emits a UserRegionBegin or UserRegionEnd event. // //go:linkname trace_userLog runtime/trace.userLog func trace_userLog(id uint64, category, message string) { @@ -696,7 +668,7 @@ func trace_userLog(id uint64, category, message string) { // Need to do this check because the caller won't have it. return } - tl.eventWriter(traceGoRunning, traceProcRunning).commit(traceEvUserLog, traceArg(id), tl.string(category), tl.uniqueString(message), tl.stack(3)) + tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvUserLog, traceArg(id), tl.string(category), tl.uniqueString(message), tl.stack(3)) traceRelease(tl) } @@ -718,16 +690,18 @@ func traceThreadDestroy(mp *m) { // as well. seq := mp.trace.seqlock.Add(1) if debugTraceReentrancy && seq%2 != 1 { - throw("bad use of trace.seqlock or tracer is reentrant") + throw("bad use of trace.seqlock") } systemstack(func() { lock(&trace.lock) for i := range mp.trace.buf { - if mp.trace.buf[i] != nil { - // N.B. traceBufFlush accepts a generation, but it - // really just cares about gen%2. - traceBufFlush(mp.trace.buf[i], uintptr(i)) - mp.trace.buf[i] = nil + for exp, buf := range mp.trace.buf[i] { + if buf != nil { + // N.B. traceBufFlush accepts a generation, but it + // really just cares about gen%2. + traceBufFlush(buf, uintptr(i)) + mp.trace.buf[i][exp] = nil + } } } unlock(&trace.lock) diff --git a/src/runtime/tracestatus.go b/src/runtime/tracestatus.go index 77ccdd13..425ac37b 100644 --- a/src/runtime/tracestatus.go +++ b/src/runtime/tracestatus.go @@ -46,6 +46,11 @@ const ( ) // writeGoStatus emits a GoStatus event as well as any active ranges on the goroutine. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (w traceWriter) writeGoStatus(goid uint64, mid int64, status traceGoStatus, markAssist bool, stackID uint64) traceWriter { // The status should never be bad. Some invariant must have been violated. if status == traceGoBad { @@ -71,6 +76,11 @@ func (w traceWriter) writeGoStatus(goid uint64, mid int64, status traceGoStatus, // // The caller must fully own pp and it must be prevented from transitioning (e.g. this can be // called by a forEachP callback or from a STW). +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (w traceWriter) writeProcStatusForP(pp *p, inSTW bool) traceWriter { if !pp.trace.acquireStatus(w.gen) { return w @@ -106,6 +116,11 @@ func (w traceWriter) writeProcStatusForP(pp *p, inSTW bool) traceWriter { // // The caller must have taken ownership of a P's status writing, and the P must be // prevented from transitioning. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (w traceWriter) writeProcStatus(pid uint64, status traceProcStatus, inSweep bool) traceWriter { // The status should never be bad. Some invariant must have been violated. if status == traceProcBad { @@ -126,6 +141,11 @@ func (w traceWriter) writeProcStatus(pid uint64, status traceProcStatus, inSweep // goStatusToTraceGoStatus translates the internal status to tracGoStatus. // // status must not be _Gdead or any status whose name has the suffix "_unused." +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func goStatusToTraceGoStatus(status uint32, wr waitReason) traceGoStatus { // N.B. Ignore the _Gscan bit. We don't model it in the tracer. var tgs traceGoStatus @@ -177,6 +197,11 @@ type traceSchedResourceState struct { } // acquireStatus acquires the right to emit a Status event for the scheduling resource. +// +// nosplit because it's part of writing an event for an M, which must not +// have any stack growth. +// +//go:nosplit func (r *traceSchedResourceState) acquireStatus(gen uintptr) bool { if !r.statusTraced[gen%3].CompareAndSwap(0, 1) { return false diff --git a/src/runtime/tracetime.go b/src/runtime/tracetime.go index 57101241..d5ee2b07 100644 --- a/src/runtime/tracetime.go +++ b/src/runtime/tracetime.go @@ -47,7 +47,8 @@ type traceTime uint64 // the timestamp from is specific to tracing, and shouldn't be mixed with other // clock sources. // -// nosplit because it's called from exitsyscall, which is nosplit. +// nosplit because it's called from exitsyscall and various trace writing functions, +// which are nosplit. // // traceClockNow is called by golang.org/x/exp/trace using linkname. // diff --git a/src/runtime/tracetype.go b/src/runtime/tracetype.go index b27a6909..d9e340f6 100644 --- a/src/runtime/tracetype.go +++ b/src/runtime/tracetype.go @@ -43,7 +43,7 @@ func (t *traceTypeTable) dump(gen uintptr) { t.tab.reset() } -func dumpTypesRec(node *traceMapNode, w traceExpWriter) traceExpWriter { +func dumpTypesRec(node *traceMapNode, w traceWriter) traceWriter { typ := (*abi.Type)(*(*unsafe.Pointer)(unsafe.Pointer(&node.data[0]))) typName := toRType(typ).string() diff --git a/src/runtime/type.go b/src/runtime/type.go index 20134075..1edf9c9d 100644 --- a/src/runtime/type.go +++ b/src/runtime/type.go @@ -8,6 +8,9 @@ package runtime import ( "internal/abi" + "internal/goarch" + "internal/goexperiment" + "internal/runtime/atomic" "unsafe" ) @@ -72,6 +75,184 @@ func (t rtype) pkgpath() string { return "" } +// getGCMask returns the pointer/nonpointer bitmask for type t. +// +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nosplit +func getGCMask(t *_type) *byte { + if t.TFlag&abi.TFlagGCMaskOnDemand != 0 { + // Split the rest into getGCMaskOnDemand so getGCMask itself is inlineable. + return getGCMaskOnDemand(t) + } + return t.GCData +} + +// inProgress is a byte whose address is a sentinel indicating that +// some thread is currently building the GC bitmask for a type. +var inProgress byte + +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nosplit +func getGCMaskOnDemand(t *_type) *byte { + // For large types, GCData doesn't point directly to a bitmask. + // Instead it points to a pointer to a bitmask, and the runtime + // is responsible for (on first use) creating the bitmask and + // storing a pointer to it in that slot. + // TODO: we could use &t.GCData as the slot, but types are + // in read-only memory currently. + addr := unsafe.Pointer(t.GCData) + + if GOOS == "aix" { + addr = add(addr, firstmoduledata.data-aixStaticDataBase) + } + + for { + p := (*byte)(atomic.Loadp(addr)) + switch p { + default: // Already built. + return p + case &inProgress: // Someone else is currently building it. + // Just wait until the builder is done. + // We can't block here, so spinning while having + // the OS thread yield is about the best we can do. + osyield() + continue + case nil: // Not built yet. + // Attempt to get exclusive access to build it. + if !atomic.Casp1((*unsafe.Pointer)(addr), nil, unsafe.Pointer(&inProgress)) { + continue + } + + // Build gcmask for this type. + bytes := goarch.PtrSize * divRoundUp(t.PtrBytes/goarch.PtrSize, 8*goarch.PtrSize) + p = (*byte)(persistentalloc(bytes, goarch.PtrSize, &memstats.other_sys)) + systemstack(func() { + buildGCMask(t, bitCursor{ptr: p, n: 0}) + }) + + // Store the newly-built gcmask for future callers. + atomic.StorepNoWB(addr, unsafe.Pointer(p)) + return p + } + } +} + +// A bitCursor is a simple cursor to memory to which we +// can write a set of bits. +type bitCursor struct { + ptr *byte // base of region + n uintptr // cursor points to bit n of region +} + +// Write to b cnt bits starting at bit 0 of data. +// Requires cnt>0. +func (b bitCursor) write(data *byte, cnt uintptr) { + // Starting byte for writing. + p := addb(b.ptr, b.n/8) + + // Note: if we're starting halfway through a byte, we load the + // existing lower bits so we don't clobber them. + n := b.n % 8 // # of valid bits in buf + buf := uintptr(*p) & (1< 8 { + // Read 8 more bits, now buf has 8-15 valid bits in it. + buf |= uintptr(*data) << n + n += 8 + data = addb(data, 1) + cnt -= 8 + // Write 8 of the buffered bits out. + *p = byte(buf) + buf >>= 8 + n -= 8 + p = addb(p, 1) + } + // Read remaining bits. + buf |= (uintptr(*data) & (1< 8 { + *p = byte(buf) + buf >>= 8 + n -= 8 + p = addb(p, 1) + } + *p &^= 1< t.Size_/2 { + // Avoid recursive call for field type that + // is larger than half of the parent type. + // There can be only one. + bigField = f + continue + } + buildGCMask(ft, dst.offset(f.Offset/goarch.PtrSize)) + } + if bigField.Typ != nil { + // Note: this case causes bits to be written out of order. + t = bigField.Typ + dst = dst.offset(bigField.Offset / goarch.PtrSize) + goto top + } + default: + throw("unexpected kind") + } +} + // reflectOffs holds type offsets defined at run time by the reflect package. // // When a type is defined at run time, its *rtype data lives on the heap. @@ -106,15 +287,6 @@ func reflectOffsUnlock() { unlock(&reflectOffs.lock) } -// resolveNameOff should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname resolveNameOff func resolveNameOff(ptrInModule unsafe.Pointer, off nameOff) name { if off == 0 { return name{} @@ -149,15 +321,6 @@ func (t rtype) nameOff(off nameOff) name { return resolveNameOff(unsafe.Pointer(t.Type), off) } -// resolveTypeOff should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname resolveTypeOff func resolveTypeOff(ptrInModule unsafe.Pointer, off typeOff) *_type { if off == 0 || off == -1 { // -1 is the sentinel value for unreachable code. @@ -235,8 +398,6 @@ type uncommontype = abi.UncommonType type interfacetype = abi.InterfaceType -type maptype = abi.MapType - type arraytype = abi.ArrayType type chantype = abi.ChanType @@ -439,8 +600,13 @@ func typesEqual(t, v *_type, seen map[_typePair]struct{}) bool { } return true case abi.Map: - mt := (*maptype)(unsafe.Pointer(t)) - mv := (*maptype)(unsafe.Pointer(v)) + if goexperiment.SwissMap { + mt := (*abi.SwissMapType)(unsafe.Pointer(t)) + mv := (*abi.SwissMapType)(unsafe.Pointer(v)) + return typesEqual(mt.Key, mv.Key, seen) && typesEqual(mt.Elem, mv.Elem, seen) + } + mt := (*abi.OldMapType)(unsafe.Pointer(t)) + mv := (*abi.OldMapType)(unsafe.Pointer(v)) return typesEqual(mt.Key, mv.Key, seen) && typesEqual(mt.Elem, mv.Elem, seen) case abi.Pointer: pt := (*ptrtype)(unsafe.Pointer(t)) diff --git a/src/runtime/unsafe.go b/src/runtime/unsafe.go index ca428b56..185eae0f 100644 --- a/src/runtime/unsafe.go +++ b/src/runtime/unsafe.go @@ -5,7 +5,8 @@ package runtime import ( - "runtime/internal/math" + "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) @@ -52,21 +53,21 @@ func panicunsafestringnilptr() { // Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice func unsafeslice(et *_type, ptr unsafe.Pointer, len int) { if len < 0 { - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } if et.Size_ == 0 { if ptr == nil && len > 0 { - panicunsafeslicenilptr1(getcallerpc()) + panicunsafeslicenilptr1(sys.GetCallerPC()) } } mem, overflow := math.MulUintptr(et.Size_, uintptr(len)) if overflow || mem > -uintptr(ptr) { if ptr == nil { - panicunsafeslicenilptr1(getcallerpc()) + panicunsafeslicenilptr1(sys.GetCallerPC()) } - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } } @@ -74,7 +75,7 @@ func unsafeslice(et *_type, ptr unsafe.Pointer, len int) { func unsafeslice64(et *_type, ptr unsafe.Pointer, len64 int64) { len := int(len64) if int64(len) != len64 { - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } unsafeslice(et, ptr, len) } @@ -92,7 +93,7 @@ func unsafeslicecheckptr(et *_type, ptr unsafe.Pointer, len64 int64) { func panicunsafeslicelen() { // This is called only from compiler-generated code, so we can get the // source of the panic. - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } //go:yeswritebarrierrec @@ -104,7 +105,7 @@ func panicunsafeslicelen1(pc uintptr) { func panicunsafeslicenilptr() { // This is called only from compiler-generated code, so we can get the // source of the panic. - panicunsafeslicenilptr1(getcallerpc()) + panicunsafeslicenilptr1(sys.GetCallerPC()) } //go:yeswritebarrierrec diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go index 45236157..72b17ce4 100644 --- a/src/runtime/vdso_linux.go +++ b/src/runtime/vdso_linux.go @@ -97,6 +97,8 @@ type vdsoInfo struct { verdef *elfVerdef } +var vdsoLoadStart, vdsoLoadEnd uintptr + // see vdso_linux_*.go for vdsoSymbolKeys[] and vdso*Sym vars func vdsoInitFromSysinfoEhdr(info *vdsoInfo, hdr *elfEhdr) { @@ -116,6 +118,8 @@ func vdsoInitFromSysinfoEhdr(info *vdsoInfo, hdr *elfEhdr) { if !foundVaddr { foundVaddr = true info.loadOffset = info.loadAddr + uintptr(pt.p_offset-pt.p_vaddr) + vdsoLoadStart = info.loadOffset + vdsoLoadEnd = info.loadOffset + uintptr(pt.p_memsz) } case _PT_DYNAMIC: @@ -285,11 +289,5 @@ func vdsoauxv(tag, val uintptr) { // //go:nosplit func inVDSOPage(pc uintptr) bool { - for _, k := range vdsoSymbolKeys { - if *k.ptr != 0 { - page := *k.ptr &^ (physPageSize - 1) - return pc >= page && pc < page+physPageSize - } - } - return false + return pc >= vdsoLoadStart && pc < vdsoLoadEnd } diff --git a/src/runtime/vdso_linux_amd64.go b/src/runtime/vdso_linux_amd64.go index 9c564091..8a89771c 100644 --- a/src/runtime/vdso_linux_amd64.go +++ b/src/runtime/vdso_linux_amd64.go @@ -17,11 +17,13 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__vdso_gettimeofday", 0x315ca59, 0xb01bca00, &vdsoGettimeofdaySym}, {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym}, + {"__vdso_getrandom", 0x25425d, 0x84a559bf, &vdsoGetrandomSym}, } var ( vdsoGettimeofdaySym uintptr vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) // vdsoGettimeofdaySym is accessed from the syscall package. diff --git a/src/runtime/vdso_linux_arm64.go b/src/runtime/vdso_linux_arm64.go index f5959525..21f875d0 100644 --- a/src/runtime/vdso_linux_arm64.go +++ b/src/runtime/vdso_linux_arm64.go @@ -15,7 +15,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.39", 0x75fcb89} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, + {"__kernel_getrandom", 0x9800c0d, 0x540d4e24, &vdsoGetrandomSym}, } -// initialize to fall back to syscall -var vdsoClockgettimeSym uintptr = 0 +var ( + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr +) diff --git a/src/runtime/vdso_linux_loong64.go b/src/runtime/vdso_linux_loong64.go index e00ef952..37ee02dc 100644 --- a/src/runtime/vdso_linux_loong64.go +++ b/src/runtime/vdso_linux_loong64.go @@ -19,9 +19,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_5.10", 0xae78f70} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym}, + {"__vdso_getrandom", 0x25425d, 0x84a559bf, &vdsoGetrandomSym}, } -// initialize to fall back to syscall var ( - vdsoClockgettimeSym uintptr = 0 + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) diff --git a/src/runtime/vdso_linux_ppc64x.go b/src/runtime/vdso_linux_ppc64x.go index 09c8d9d2..939da7b5 100644 --- a/src/runtime/vdso_linux_ppc64x.go +++ b/src/runtime/vdso_linux_ppc64x.go @@ -16,9 +16,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.15", 0x75fcba5} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, + {"__kernel_getrandom", 0x9800c0d, 0x540d4e24, &vdsoGetrandomSym}, } -// initialize with vsyscall fallbacks var ( - vdsoClockgettimeSym uintptr = 0 + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) diff --git a/src/runtime/vdso_linux_s390x.go b/src/runtime/vdso_linux_s390x.go index 970ecd3c..113152ff 100644 --- a/src/runtime/vdso_linux_s390x.go +++ b/src/runtime/vdso_linux_s390x.go @@ -16,9 +16,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.29", 0x75fcbb9} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, + {"__kernel_getrandom", 0x9800c0d, 0x540d4e24, &vdsoGetrandomSym}, } -// initialize with vsyscall fallbacks var ( - vdsoClockgettimeSym uintptr = 0 + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) diff --git a/src/runtime/vgetrandom_linux.go b/src/runtime/vgetrandom_linux.go new file mode 100644 index 00000000..a6ec4b70 --- /dev/null +++ b/src/runtime/vgetrandom_linux.go @@ -0,0 +1,112 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (amd64 || arm64 || arm64be || ppc64 || ppc64le || loong64 || s390x) + +package runtime + +import ( + "internal/cpu" + "unsafe" +) + +//go:noescape +func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int + +var vgetrandomAlloc struct { + states []uintptr + statesLock mutex + stateSize uintptr + mmapProt int32 + mmapFlags int32 +} + +func vgetrandomInit() { + if vdsoGetrandomSym == 0 { + return + } + + var params struct { + SizeOfOpaqueState uint32 + MmapProt uint32 + MmapFlags uint32 + reserved [13]uint32 + } + if vgetrandom1(nil, 0, 0, uintptr(unsafe.Pointer(¶ms)), ^uintptr(0)) != 0 { + return + } + vgetrandomAlloc.stateSize = uintptr(params.SizeOfOpaqueState) + vgetrandomAlloc.mmapProt = int32(params.MmapProt) + vgetrandomAlloc.mmapFlags = int32(params.MmapFlags) + + lockInit(&vgetrandomAlloc.statesLock, lockRankLeafRank) +} + +func vgetrandomGetState() uintptr { + lock(&vgetrandomAlloc.statesLock) + if len(vgetrandomAlloc.states) == 0 { + num := uintptr(ncpu) // Just a reasonable size hint to start. + stateSizeCacheAligned := (vgetrandomAlloc.stateSize + cpu.CacheLineSize - 1) &^ (cpu.CacheLineSize - 1) + allocSize := (num*stateSizeCacheAligned + physPageSize - 1) &^ (physPageSize - 1) + num = (physPageSize / stateSizeCacheAligned) * (allocSize / physPageSize) + p, err := mmap(nil, allocSize, vgetrandomAlloc.mmapProt, vgetrandomAlloc.mmapFlags, -1, 0) + if err != 0 { + unlock(&vgetrandomAlloc.statesLock) + return 0 + } + newBlock := uintptr(p) + if vgetrandomAlloc.states == nil { + vgetrandomAlloc.states = make([]uintptr, 0, num) + } + for i := uintptr(0); i < num; i++ { + if (newBlock&(physPageSize-1))+vgetrandomAlloc.stateSize > physPageSize { + newBlock = (newBlock + physPageSize - 1) &^ (physPageSize - 1) + } + vgetrandomAlloc.states = append(vgetrandomAlloc.states, newBlock) + newBlock += stateSizeCacheAligned + } + } + state := vgetrandomAlloc.states[len(vgetrandomAlloc.states)-1] + vgetrandomAlloc.states = vgetrandomAlloc.states[:len(vgetrandomAlloc.states)-1] + unlock(&vgetrandomAlloc.statesLock) + return state +} + +func vgetrandomPutState(state uintptr) { + lock(&vgetrandomAlloc.statesLock) + vgetrandomAlloc.states = append(vgetrandomAlloc.states, state) + unlock(&vgetrandomAlloc.statesLock) +} + +// This is exported for use in internal/syscall/unix as well as x/sys/unix. +// +//go:linkname vgetrandom +func vgetrandom(p []byte, flags uint32) (ret int, supported bool) { + if vgetrandomAlloc.stateSize == 0 { + return -1, false + } + + // We use getg().m instead of acquirem() here, because always taking + // the lock is slightly more expensive than not always taking the lock. + // However, we *do* require that m doesn't migrate elsewhere during the + // execution of the vDSO. So, we exploit two details: + // 1) Asynchronous preemption is aborted when PC is in the runtime. + // 2) Most of the time, this function only calls vgetrandom1(), which + // does not have a preamble that synchronously preempts. + // We do need to take the lock when getting a new state for m, but this + // is very much the slow path, in the sense that it only ever happens + // once over the entire lifetime of an m. So, a simple getg().m suffices. + mp := getg().m + + if mp.vgetrandomState == 0 { + mp.locks++ + state := vgetrandomGetState() + mp.locks-- + if state == 0 { + return -1, false + } + mp.vgetrandomState = state + } + return vgetrandom1(unsafe.SliceData(p), uintptr(len(p)), flags, mp.vgetrandomState, vgetrandomAlloc.stateSize), true +} diff --git a/src/runtime/vgetrandom_unsupported.go b/src/runtime/vgetrandom_unsupported.go new file mode 100644 index 00000000..070392cf --- /dev/null +++ b/src/runtime/vgetrandom_unsupported.go @@ -0,0 +1,18 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !(linux && (amd64 || arm64 || arm64be || ppc64 || ppc64le || loong64 || s390x)) + +package runtime + +import _ "unsafe" + +//go:linkname vgetrandom +func vgetrandom(p []byte, flags uint32) (ret int, supported bool) { + return -1, false +} + +func vgetrandomPutState(state uintptr) {} + +func vgetrandomInit() {} diff --git a/src/slices/example_test.go b/src/slices/example_test.go index cb601ada..f775b88f 100644 --- a/src/slices/example_test.go +++ b/src/slices/example_test.go @@ -310,7 +310,7 @@ func ExampleSortStableFunc() { {"Bob", 24}, {"Alice", 55}, } - // Stable sort by name, keeping age ordering of Alices intact + // Stable sort by name, keeping age ordering of Alice intact slices.SortStableFunc(people, func(a, b Person) int { return strings.Compare(a.Name, b.Name) }) @@ -325,9 +325,11 @@ func ExampleClone() { fmt.Println(clone) clone[2] = 10 fmt.Println(numbers) + fmt.Println(clone) // Output: // [0 42 -10 8] // [0 42 -10 8] + // [0 42 10 8] } func ExampleGrow() { @@ -385,6 +387,133 @@ func ExampleRepeat() { // [0 1 2 3 0 1 2 3] } +func ExampleAll() { + names := []string{"Alice", "Bob", "Vera"} + for i, v := range slices.All(names) { + fmt.Println(i, ":", v) + } + // Output: + // 0 : Alice + // 1 : Bob + // 2 : Vera +} + +func ExampleBackward() { + names := []string{"Alice", "Bob", "Vera"} + for i, v := range slices.Backward(names) { + fmt.Println(i, ":", v) + } + // Output: + // 2 : Vera + // 1 : Bob + // 0 : Alice +} + +func ExampleValues() { + names := []string{"Alice", "Bob", "Vera"} + for v := range slices.Values(names) { + fmt.Println(v) + } + // Output: + // Alice + // Bob + // Vera +} + +func ExampleAppendSeq() { + seq := func(yield func(int) bool) { + for i := 0; i < 10; i += 2 { + if !yield(i) { + return + } + } + } + + s := slices.AppendSeq([]int{1, 2}, seq) + fmt.Println(s) + // Output: + // [1 2 0 2 4 6 8] +} + +func ExampleCollect() { + seq := func(yield func(int) bool) { + for i := 0; i < 10; i += 2 { + if !yield(i) { + return + } + } + } + + s := slices.Collect(seq) + fmt.Println(s) + // Output: + // [0 2 4 6 8] +} + +func ExampleSorted() { + seq := func(yield func(int) bool) { + flag := -1 + for i := 0; i < 10; i += 2 { + flag = -flag + if !yield(i * flag) { + return + } + } + } + + s := slices.Sorted(seq) + fmt.Println(s) + fmt.Println(slices.IsSorted(s)) + // Output: + // [-6 -2 0 4 8] + // true +} + +func ExampleSortedFunc() { + seq := func(yield func(int) bool) { + flag := -1 + for i := 0; i < 10; i += 2 { + flag = -flag + if !yield(i * flag) { + return + } + } + } + + sortFunc := func(a, b int) int { + return cmp.Compare(b, a) // the comparison is being done in reverse + } + + s := slices.SortedFunc(seq, sortFunc) + fmt.Println(s) + // Output: + // [8 4 0 -2 -6] +} + +func ExampleSortedStableFunc() { + type Person struct { + Name string + Age int + } + + people := []Person{ + {"Gopher", 13}, + {"Alice", 20}, + {"Bob", 5}, + {"Vera", 24}, + {"Zac", 20}, + } + + sortFunc := func(x, y Person) int { + return cmp.Compare(x.Age, y.Age) + } + + s := slices.SortedStableFunc(slices.Values(people), sortFunc) + fmt.Println(s) + // Output: + // [{Bob 5} {Gopher 13} {Alice 20} {Zac 20} {Vera 24}] +} + func ExampleChunk() { type Person struct { Name string diff --git a/src/slices/slices.go b/src/slices/slices.go index b3cd4e2c..32029cd8 100644 --- a/src/slices/slices.go +++ b/src/slices/slices.go @@ -128,8 +128,8 @@ func ContainsFunc[S ~[]E, E any](s S, f func(E) bool) bool { // returning the modified slice. // The elements at s[i:] are shifted up to make room. // In the returned slice r, r[i] == v[0], -// and r[i+len(v)] == value originally at r[i]. -// Insert panics if i is out of range. +// and, if i < len(s), r[i+len(v)] == value originally at r[i]. +// Insert panics if i > len(s). // This function is O(len(s) + len(v)). func Insert[S ~[]E, E any](s S, i int, v ...E) S { _ = s[i:] // bounds check @@ -346,8 +346,13 @@ func Replace[S ~[]E, E any](s S, i, j int, v ...E) S { // The elements are copied using assignment, so this is a shallow clone. // The result may have additional unused capacity. func Clone[S ~[]E, E any](s S) S { - // The s[:0:0] preserves nil in case it matters. - return append(s[:0:0], s...) + // Preserve nilness in case it matters. + if s == nil { + return nil + } + // Avoid s[:0:0] as it leads to unwanted liveness when cloning a + // zero-length slice of a large array; see https://go.dev/issue/68488. + return append(S{}, s...) } // Compact replaces consecutive runs of equal elements with a single copy. @@ -409,6 +414,7 @@ func Grow[S ~[]E, E any](s S, n int) S { panic("cannot be negative") } if n -= cap(s) - len(s); n > 0 { + // This expression allocates only once (see test). s = append(s[:cap(s)], make([]E, n)...)[:len(s)] } return s @@ -434,7 +440,7 @@ func rotateRight[E any](s []E, r int) { rotateLeft(s, len(s)-r) } -// overlaps reports whether the memory ranges a[0:len(a)] and b[0:len(b)] overlap. +// overlaps reports whether the memory ranges a[:len(a)] and b[:len(b)] overlap. func overlaps[E any](a, b []E) bool { if len(a) == 0 || len(b) == 0 { return false @@ -444,7 +450,7 @@ func overlaps[E any](a, b []E) bool { return false } // TODO: use a runtime/unsafe facility once one becomes available. See issue 12445. - // Also see crypto/internal/alias/alias.go:AnyOverlap + // Also see crypto/internal/fips140/alias/alias.go:AnyOverlap return uintptr(unsafe.Pointer(&a[0])) <= uintptr(unsafe.Pointer(&b[len(b)-1]))+(elemSize-1) && uintptr(unsafe.Pointer(&b[0])) <= uintptr(unsafe.Pointer(&a[len(a)-1]))+(elemSize-1) } @@ -478,6 +484,9 @@ func Concat[S ~[]E, E any](slices ...S) S { panic("len out of range") } } + // Use Grow, not make, to round up to the size class: + // the extra space is otherwise unused and helps + // callers that append a few elements to the result. newslice := Grow[S](nil, size) for _, s := range slices { newslice = append(newslice, s...) @@ -496,11 +505,12 @@ func Repeat[S ~[]E, E any](x S, count int) S { } const maxInt = ^uint(0) >> 1 - if hi, lo := bits.Mul(uint(len(x)), uint(count)); hi > 0 || lo > maxInt { + hi, lo := bits.Mul(uint(len(x)), uint(count)) + if hi > 0 || lo > maxInt { panic("the result of (len(x) * count) overflows") } - newslice := make(S, len(x)*count) + newslice := make(S, int(lo)) // lo = len(x) * count n := copy(newslice, x) for n < len(newslice) { n += copy(newslice[n:], newslice[:n]) diff --git a/src/slices/slices_test.go b/src/slices/slices_test.go index 68c8a3ad..4ced7c07 100644 --- a/src/slices/slices_test.go +++ b/src/slices/slices_test.go @@ -6,12 +6,15 @@ package slices_test import ( "cmp" + "internal/asan" + "internal/msan" "internal/race" "internal/testenv" "math" . "slices" "strings" "testing" + "unsafe" ) var equalIntTests = []struct { @@ -496,7 +499,7 @@ func TestInsert(t *testing.T) { } } - if !testenv.OptimizationOff() && !race.Enabled { + if !testenv.OptimizationOff() && !race.Enabled && !asan.Enabled && !msan.Enabled { // Allocations should be amortized. const count = 50 n := testing.AllocsPerRun(10, func() { @@ -952,7 +955,7 @@ func TestGrow(t *testing.T) { } if n := testing.AllocsPerRun(100, func() { _ = Grow(s2, cap(s2)-len(s2)+1) }); n != 1 { errorf := t.Errorf - if race.Enabled || testenv.OptimizationOff() { + if race.Enabled || msan.Enabled || asan.Enabled || testenv.OptimizationOff() { errorf = t.Logf // this allocates multiple times in race detector mode } errorf("Grow should allocate once when given insufficient capacity; allocated %v times", n) @@ -1009,7 +1012,7 @@ func TestReverse(t *testing.T) { singleton := []string{"one"} Reverse(singleton) if want := []string{"one"}; !Equal(singleton, want) { - t.Errorf("Reverse(singeleton) = %v, want %v", singleton, want) + t.Errorf("Reverse(singleton) = %v, want %v", singleton, want) } Reverse[[]string](nil) @@ -1313,7 +1316,7 @@ func TestConcat(t *testing.T) { _ = sink if allocs > 1 { errorf := t.Errorf - if testenv.OptimizationOff() || race.Enabled { + if testenv.OptimizationOff() || race.Enabled || asan.Enabled || msan.Enabled { errorf = t.Logf } errorf("Concat(%v) allocated %v times; want 1", tc.s, allocs) @@ -1450,3 +1453,12 @@ func TestRepeatPanics(t *testing.T) { } } } + +func TestIssue68488(t *testing.T) { + s := make([]int, 3) + clone := Clone(s[1:1]) + switch unsafe.SliceData(clone) { + case &s[0], &s[1], &s[2]: + t.Error("clone keeps alive s due to array overlap") + } +} diff --git a/src/slices/sort.go b/src/slices/sort.go index f713ffe0..4f66e7bb 100644 --- a/src/slices/sort.go +++ b/src/slices/sort.go @@ -180,8 +180,8 @@ type xorshift uint64 func (r *xorshift) Next() uint64 { *r ^= *r << 13 - *r ^= *r >> 17 - *r ^= *r << 5 + *r ^= *r >> 7 + *r ^= *r << 17 return uint64(*r) } diff --git a/src/sort/example_keys_test.go b/src/sort/example_keys_test.go index 648f919e..5e25f0d3 100644 --- a/src/sort/example_keys_test.go +++ b/src/sort/example_keys_test.go @@ -60,7 +60,7 @@ var planets = []Planet{ {"Mars", 0.107, 1.5}, } -// ExampleSortKeys demonstrates a technique for sorting a struct type using programmable sort criteria. +// Example_sortKeys demonstrates a technique for sorting a struct type using programmable sort criteria. func Example_sortKeys() { // Closures that order the Planet structure. name := func(p1, p2 *Planet) bool { diff --git a/src/sort/example_multi_test.go b/src/sort/example_multi_test.go index 93f2d3ec..c21050b7 100644 --- a/src/sort/example_multi_test.go +++ b/src/sort/example_multi_test.go @@ -87,7 +87,7 @@ var changes = []Change{ {"gri", "Smalltalk", 80}, } -// ExampleMultiKeys demonstrates a technique for sorting a struct type using different +// Example_sortMultiKeys demonstrates a technique for sorting a struct type using different // sets of multiple fields in the comparison. We chain together "Less" functions, each of // which compares a single field. func Example_sortMultiKeys() { diff --git a/src/sort/example_search_test.go b/src/sort/example_search_test.go index eadac9a7..f621dfb4 100644 --- a/src/sort/example_search_test.go +++ b/src/sort/example_search_test.go @@ -93,3 +93,20 @@ func ExampleSearchInts() { // found 2 at index 1 in [1 2 3 4 6 7 8] // 5 not found, can be inserted at index 4 in [1 2 3 4 6 7 8] } + +// This example demonstrates searching for string in a list sorted in ascending order. +func ExampleSearchStrings() { + a := []string{"apple", "banana", "cherry", "date", "fig", "grape"} + + x := "banana" + i := sort.SearchStrings(a, x) + fmt.Printf("found %s at index %d in %v\n", x, i, a) + + x = "coconut" + i = sort.SearchStrings(a, x) + fmt.Printf("%s not found, can be inserted at index %d in %v\n", x, i, a) + + // Output: + // found banana at index 1 in [apple banana cherry date fig grape] + // coconut not found, can be inserted at index 3 in [apple banana cherry date fig grape] +} diff --git a/src/sort/example_test.go b/src/sort/example_test.go index 1f85dbcb..32eb73c8 100644 --- a/src/sort/example_test.go +++ b/src/sort/example_test.go @@ -86,6 +86,34 @@ func ExampleSlice() { // By age: [{Gopher 7} {Vera 24} {Alice 55} {Bob 75}] } +func ExampleSliceIsSorted() { + numbers := []int{1, 2, 3, 4, 5, 6} + + isSortedAsc := sort.SliceIsSorted(numbers, func(i, j int) bool { + return numbers[i] < numbers[j] + }) + fmt.Printf("%v sorted ascending: %t\n", numbers, isSortedAsc) + + numbersDesc := []int{6, 5, 4, 3, 2, 1} + + isSortedDesc := sort.SliceIsSorted(numbersDesc, func(i, j int) bool { + return numbersDesc[i] > numbersDesc[j] + }) + fmt.Printf("%v sorted descending: %t\n", numbers, isSortedDesc) + + unsortedNumbers := []int{1, 3, 2, 4, 5} + + isSortedUnsorted := sort.SliceIsSorted(unsortedNumbers, func(i, j int) bool { + return unsortedNumbers[i] < unsortedNumbers[j] + }) + fmt.Printf("%v unsorted slice sorted: %t\n", unsortedNumbers, isSortedUnsorted) + + // Output: + // [1 2 3 4 5 6] sorted ascending: true + // [1 2 3 4 5 6] sorted descending: true + // [1 3 2 4 5] unsorted slice sorted: false +} + func ExampleSliceStable() { people := []struct { diff --git a/src/sort/sort.go b/src/sort/sort.go index 6db161f0..b27ecabd 100644 --- a/src/sort/sort.go +++ b/src/sort/sort.go @@ -7,7 +7,10 @@ // Package sort provides primitives for sorting slices and user-defined collections. package sort -import "math/bits" +import ( + "math/bits" + "slices" +) // An implementation of Interface can be sorted by the routines in this package. // The methods refer to elements of the underlying collection by integer index. @@ -64,8 +67,8 @@ type xorshift uint64 func (r *xorshift) Next() uint64 { *r ^= *r << 13 - *r ^= *r >> 17 - *r ^= *r << 5 + *r ^= *r >> 7 + *r ^= *r << 17 return uint64(*r) } @@ -162,34 +165,34 @@ func (x StringSlice) Sort() { Sort(x) } // Ints sorts a slice of ints in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.Sort]. -func Ints(x []int) { intsImpl(x) } +func Ints(x []int) { slices.Sort(x) } // Float64s sorts a slice of float64s in increasing order. // Not-a-number (NaN) values are ordered before other values. // // Note: as of Go 1.22, this function simply calls [slices.Sort]. -func Float64s(x []float64) { float64sImpl(x) } +func Float64s(x []float64) { slices.Sort(x) } // Strings sorts a slice of strings in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.Sort]. -func Strings(x []string) { stringsImpl(x) } +func Strings(x []string) { slices.Sort(x) } // IntsAreSorted reports whether the slice x is sorted in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.IsSorted]. -func IntsAreSorted(x []int) bool { return intsAreSortedImpl(x) } +func IntsAreSorted(x []int) bool { return slices.IsSorted(x) } // Float64sAreSorted reports whether the slice x is sorted in increasing order, // with not-a-number (NaN) values before any other values. // // Note: as of Go 1.22, this function simply calls [slices.IsSorted]. -func Float64sAreSorted(x []float64) bool { return float64sAreSortedImpl(x) } +func Float64sAreSorted(x []float64) bool { return slices.IsSorted(x) } // StringsAreSorted reports whether the slice x is sorted in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.IsSorted]. -func StringsAreSorted(x []string) bool { return stringsAreSortedImpl(x) } +func StringsAreSorted(x []string) bool { return slices.IsSorted(x) } // Notes on stable sorting: // The used algorithms are simple and provable correct on all input and use diff --git a/src/sort/sort_impl_120.go b/src/sort/sort_impl_120.go deleted file mode 100644 index 5980da67..00000000 --- a/src/sort/sort_impl_120.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.21 - -package sort - -func intsImpl(x []int) { Sort(IntSlice(x)) } -func float64sImpl(x []float64) { Sort(Float64Slice(x)) } -func stringsImpl(x []string) { Sort(StringSlice(x)) } - -func intsAreSortedImpl(x []int) bool { return IsSorted(IntSlice(x)) } -func float64sAreSortedImpl(x []float64) bool { return IsSorted(Float64Slice(x)) } -func stringsAreSortedImpl(x []string) bool { return IsSorted(StringSlice(x)) } diff --git a/src/sort/sort_impl_go121.go b/src/sort/sort_impl_go121.go deleted file mode 100644 index 0a6a6a62..00000000 --- a/src/sort/sort_impl_go121.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.21 - -// Starting with Go 1.21, we can leverage the new generic functions from the -// slices package to implement some `sort` functions faster. However, until -// the bootstrap compiler uses Go 1.21 or later, we keep a fallback version -// in sort_impl_120.go that retains the old implementation. - -package sort - -import "slices" - -func intsImpl(x []int) { slices.Sort(x) } -func float64sImpl(x []float64) { slices.Sort(x) } -func stringsImpl(x []string) { slices.Sort(x) } - -func intsAreSortedImpl(x []int) bool { return slices.IsSorted(x) } -func float64sAreSortedImpl(x []float64) bool { return slices.IsSorted(x) } -func stringsAreSortedImpl(x []string) bool { return slices.IsSorted(x) } diff --git a/src/strconv/ftoa.go b/src/strconv/ftoa.go index 22086989..bfe26366 100644 --- a/src/strconv/ftoa.go +++ b/src/strconv/ftoa.go @@ -28,14 +28,14 @@ var float64info = floatInfo{52, 11, -1023} // value of bitSize bits (32 for float32, 64 for float64). // // The format fmt is one of -// 'b' (-ddddp±ddd, a binary exponent), -// 'e' (-d.dddde±dd, a decimal exponent), -// 'E' (-d.ddddE±dd, a decimal exponent), -// 'f' (-ddd.dddd, no exponent), -// 'g' ('e' for large exponents, 'f' otherwise), -// 'G' ('E' for large exponents, 'f' otherwise), -// 'x' (-0xd.ddddp±ddd, a hexadecimal fraction and binary exponent), or -// 'X' (-0Xd.ddddP±ddd, a hexadecimal fraction and binary exponent). +// - 'b' (-ddddp±ddd, a binary exponent), +// - 'e' (-d.dddde±dd, a decimal exponent), +// - 'E' (-d.ddddE±dd, a decimal exponent), +// - 'f' (-ddd.dddd, no exponent), +// - 'g' ('e' for large exponents, 'f' otherwise), +// - 'G' ('E' for large exponents, 'f' otherwise), +// - 'x' (-0xd.ddddp±ddd, a hexadecimal fraction and binary exponent), or +// - 'X' (-0Xd.ddddP±ddd, a hexadecimal fraction and binary exponent). // // The precision prec controls the number of digits (excluding the exponent) // printed by the 'e', 'E', 'f', 'g', 'G', 'x', and 'X' formats. @@ -44,6 +44,8 @@ var float64info = floatInfo{52, 11, -1023} // zeros are removed). // The special precision -1 uses the smallest number of digits // necessary such that ParseFloat will return f exactly. +// The exponent is written as a decimal integer; +// for all formats other than 'b', it will be at least two digits. func FormatFloat(f float64, fmt byte, prec, bitSize int) string { return string(genericFtoa(make([]byte, 0, max(prec+4, 24)), f, fmt, prec, bitSize)) } diff --git a/src/strconv/itoa.go b/src/strconv/itoa.go index 29fec41f..928b37ff 100644 --- a/src/strconv/itoa.go +++ b/src/strconv/itoa.go @@ -152,13 +152,7 @@ func formatBits(dst []byte, u uint64, base int, neg, append_ bool) (d []byte, s } else if isPowerOfTwo(base) { // Use shifts and masks instead of / and %. - // Base is a power of 2 and 2 <= base <= len(digits) where len(digits) is 36. - // The largest power of 2 below or equal to 36 is 32, which is 1 << 5; - // i.e., the largest possible shift count is 5. By &-ind that value with - // the constant 7 we tell the compiler that the shift count is always - // less than 8 which is smaller than any register width. This allows - // the compiler to generate better code for the shift operation. - shift := uint(bits.TrailingZeros(uint(base))) & 7 + shift := uint(bits.TrailingZeros(uint(base))) b := uint64(base) m := uint(base) - 1 // == 1<= b { diff --git a/src/strconv/quote.go b/src/strconv/quote.go index d626cd08..99c292a8 100644 --- a/src/strconv/quote.go +++ b/src/strconv/quote.go @@ -378,7 +378,8 @@ func QuotedPrefix(s string) (string, error) { // or backquoted Go string literal, returning the string value // that s quotes. (If s is single-quoted, it would be a Go // character literal; Unquote returns the corresponding -// one-character string.) +// one-character string. For an empty character literal +// Unquote returns the empty string.) func Unquote(s string) (string, error) { out, rem, err := unquote(s, true) if len(rem) > 0 { diff --git a/src/strings/builder_test.go b/src/strings/builder_test.go index 36fd7a77..06cd3e3b 100644 --- a/src/strings/builder_test.go +++ b/src/strings/builder_test.go @@ -6,6 +6,7 @@ package strings_test import ( "bytes" + "internal/asan" . "strings" "testing" "unicode/utf8" @@ -89,6 +90,10 @@ func TestBuilderReset(t *testing.T) { func TestBuilderGrow(t *testing.T) { for _, growLen := range []int{0, 100, 1000, 10000, 100000} { + if asan.Enabled { + t.Logf("skipping allocs check for growLen %d: extra allocs with -asan; see #70079", growLen) + continue + } p := bytes.Repeat([]byte{'a'}, growLen) allocs := testing.AllocsPerRun(100, func() { var b Builder @@ -188,6 +193,9 @@ func TestBuilderWriteByte(t *testing.T) { } func TestBuilderAllocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } // Issue 23382; verify that copyCheck doesn't force the // Builder to escape and be heap allocated. n := testing.AllocsPerRun(10000, func() { @@ -387,6 +395,9 @@ func BenchmarkBuildString_ByteBuffer(b *testing.B) { } func TestBuilderGrowSizeclasses(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } s := Repeat("a", 19) allocs := testing.AllocsPerRun(100, func() { var b Builder diff --git a/src/strings/example_test.go b/src/strings/example_test.go index bdab7ae8..08efcbf6 100644 --- a/src/strings/example_test.go +++ b/src/strings/example_test.go @@ -328,22 +328,22 @@ func ExampleTitle() { // Compare this example to the ToTitle example. fmt.Println(strings.Title("her royal highness")) fmt.Println(strings.Title("loud noises")) - fmt.Println(strings.Title("хлеб")) + fmt.Println(strings.Title("брат")) // Output: // Her Royal Highness // Loud Noises - // Хлеб + // Брат } func ExampleToTitle() { // Compare this example to the Title example. fmt.Println(strings.ToTitle("her royal highness")) fmt.Println(strings.ToTitle("loud noises")) - fmt.Println(strings.ToTitle("хлеб")) + fmt.Println(strings.ToTitle("брат")) // Output: // HER ROYAL HIGHNESS // LOUD NOISES - // ХЛЕБ + // БРАТ } func ExampleToTitleSpecial() { @@ -388,8 +388,8 @@ func ExampleToLower() { } func ExampleToLowerSpecial() { - fmt.Println(strings.ToLowerSpecial(unicode.TurkishCase, "Önnek İş")) - // Output: önnek iş + fmt.Println(strings.ToLowerSpecial(unicode.TurkishCase, "Örnek İş")) + // Output: örnek iş } func ExampleTrim() { diff --git a/src/strings/iter.go b/src/strings/iter.go new file mode 100644 index 00000000..3168e596 --- /dev/null +++ b/src/strings/iter.go @@ -0,0 +1,148 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strings + +import ( + "iter" + "unicode" + "unicode/utf8" +) + +// Lines returns an iterator over the newline-terminated lines in the string s. +// The lines yielded by the iterator include their terminating newlines. +// If s is empty, the iterator yields no lines at all. +// If s does not end in a newline, the final yielded line will not end in a newline. +// It returns a single-use iterator. +func Lines(s string) iter.Seq[string] { + return func(yield func(string) bool) { + for len(s) > 0 { + var line string + if i := IndexByte(s, '\n'); i >= 0 { + line, s = s[:i+1], s[i+1:] + } else { + line, s = s, "" + } + if !yield(line) { + return + } + } + return + } +} + +// explodeSeq returns an iterator over the runes in s. +func explodeSeq(s string) iter.Seq[string] { + return func(yield func(string) bool) { + for len(s) > 0 { + _, size := utf8.DecodeRuneInString(s) + if !yield(s[:size]) { + return + } + s = s[size:] + } + } +} + +// splitSeq is SplitSeq or SplitAfterSeq, configured by how many +// bytes of sep to include in the results (none or all). +func splitSeq(s, sep string, sepSave int) iter.Seq[string] { + if len(sep) == 0 { + return explodeSeq(s) + } + return func(yield func(string) bool) { + for { + i := Index(s, sep) + if i < 0 { + break + } + frag := s[:i+sepSave] + if !yield(frag) { + return + } + s = s[i+len(sep):] + } + yield(s) + } +} + +// SplitSeq returns an iterator over all substrings of s separated by sep. +// The iterator yields the same strings that would be returned by [Split](s, sep), +// but without constructing the slice. +// It returns a single-use iterator. +func SplitSeq(s, sep string) iter.Seq[string] { + return splitSeq(s, sep, 0) +} + +// SplitAfterSeq returns an iterator over substrings of s split after each instance of sep. +// The iterator yields the same strings that would be returned by [SplitAfter](s, sep), +// but without constructing the slice. +// It returns a single-use iterator. +func SplitAfterSeq(s, sep string) iter.Seq[string] { + return splitSeq(s, sep, len(sep)) +} + +// FieldsSeq returns an iterator over substrings of s split around runs of +// whitespace characters, as defined by [unicode.IsSpace]. +// The iterator yields the same strings that would be returned by [Fields](s), +// but without constructing the slice. +func FieldsSeq(s string) iter.Seq[string] { + return func(yield func(string) bool) { + start := -1 + for i := 0; i < len(s); { + size := 1 + r := rune(s[i]) + isSpace := asciiSpace[s[i]] != 0 + if r >= utf8.RuneSelf { + r, size = utf8.DecodeRuneInString(s[i:]) + isSpace = unicode.IsSpace(r) + } + if isSpace { + if start >= 0 { + if !yield(s[start:i]) { + return + } + start = -1 + } + } else if start < 0 { + start = i + } + i += size + } + if start >= 0 { + yield(s[start:]) + } + } +} + +// FieldsFuncSeq returns an iterator over substrings of s split around runs of +// Unicode code points satisfying f(c). +// The iterator yields the same strings that would be returned by [FieldsFunc](s), +// but without constructing the slice. +func FieldsFuncSeq(s string, f func(rune) bool) iter.Seq[string] { + return func(yield func(string) bool) { + start := -1 + for i := 0; i < len(s); { + size := 1 + r := rune(s[i]) + if r >= utf8.RuneSelf { + r, size = utf8.DecodeRuneInString(s[i:]) + } + if f(r) { + if start >= 0 { + if !yield(s[start:i]) { + return + } + start = -1 + } + } else if start < 0 { + start = i + } + i += size + } + if start >= 0 { + yield(s[start:]) + } + } +} diff --git a/src/strings/search_test.go b/src/strings/search_test.go index c01a393a..03408504 100644 --- a/src/strings/search_test.go +++ b/src/strings/search_test.go @@ -5,7 +5,7 @@ package strings_test import ( - "reflect" + "slices" . "strings" "testing" ) @@ -83,7 +83,7 @@ func TestFinderCreation(t *testing.T) { } } - if !reflect.DeepEqual(good, tc.suf) { + if !slices.Equal(good, tc.suf) { t.Errorf("boyerMoore(%q) got %v want %v", tc.pattern, good, tc.suf) } } diff --git a/src/strings/strings.go b/src/strings/strings.go index 0bd3c1c2..7eb2de63 100644 --- a/src/strings/strings.go +++ b/src/strings/strings.go @@ -10,6 +10,7 @@ package strings import ( "internal/bytealg" "internal/stringslite" + "math/bits" "unicode" "unicode/utf8" ) @@ -124,6 +125,7 @@ func IndexByte(s string, c byte) int { // If r is [utf8.RuneError], it returns the first instance of any // invalid UTF-8 byte sequence. func IndexRune(s string, r rune) int { + const haveFastIndex = bytealg.MaxBruteForce > 0 switch { case 0 <= r && r < utf8.RuneSelf: return IndexByte(s, byte(r)) @@ -137,7 +139,60 @@ func IndexRune(s string, r rune) int { case !utf8.ValidRune(r): return -1 default: - return Index(s, string(r)) + // Search for rune r using the last byte of its UTF-8 encoded form. + // The distribution of the last byte is more uniform compared to the + // first byte which has a 78% chance of being [240, 243, 244]. + rs := string(r) + last := len(rs) - 1 + i := last + fails := 0 + for i < len(s) { + if s[i] != rs[last] { + o := IndexByte(s[i+1:], rs[last]) + if o < 0 { + return -1 + } + i += o + 1 + } + // Step backwards comparing bytes. + for j := 1; j < len(rs); j++ { + if s[i-j] != rs[last-j] { + goto next + } + } + return i - last + next: + fails++ + i++ + if (haveFastIndex && fails > bytealg.Cutover(i)) && i < len(s) || + (!haveFastIndex && fails >= 4+i>>4 && i < len(s)) { + goto fallback + } + } + return -1 + + fallback: + // see comment in ../bytes/bytes.go + if haveFastIndex { + if j := bytealg.IndexString(s[i-last:], string(r)); j >= 0 { + return i + j - last + } + } else { + c0 := rs[last] + c1 := rs[last-1] + loop: + for ; i < len(s); i++ { + if s[i] == c0 && s[i-1] == c1 { + for k := 2; k < len(rs); k++ { + if s[i-k] != rs[last-k] { + continue loop + } + } + return i - last + } + } + } + return -1 } } @@ -568,10 +623,11 @@ func Repeat(s string, count int) string { if count < 0 { panic("strings: negative Repeat count") } - if len(s) > maxInt/count { + hi, lo := bits.Mul(uint(len(s)), uint(count)) + if hi > 0 || lo > uint(maxInt) { panic("strings: Repeat output length overflow") } - n := len(s) * count + n := int(lo) // lo = len(s) * count if len(s) == 0 { return "" @@ -617,13 +673,7 @@ func Repeat(s string, count int) string { b.Grow(n) b.WriteString(s) for b.Len() < n { - chunk := n - b.Len() - if chunk > b.Len() { - chunk = b.Len() - } - if chunk > chunkMax { - chunk = chunkMax - } + chunk := min(n-b.Len(), b.Len(), chunkMax) b.WriteString(b.String()[:chunk]) } return b.String() diff --git a/src/strings/strings_test.go b/src/strings/strings_test.go index 4bd3a3c2..80673806 100644 --- a/src/strings/strings_test.go +++ b/src/strings/strings_test.go @@ -8,9 +8,10 @@ import ( "bytes" "fmt" "io" + "iter" "math" "math/rand" - "reflect" + "slices" "strconv" . "strings" "testing" @@ -19,16 +20,35 @@ import ( "unsafe" ) -func eq(a, b []string) bool { - if len(a) != len(b) { - return false +func collect(t *testing.T, seq iter.Seq[string]) []string { + out := slices.Collect(seq) + out1 := slices.Collect(seq) + if !slices.Equal(out, out1) { + t.Fatalf("inconsistent seq:\n%s\n%s", out, out1) } - for i := 0; i < len(a); i++ { - if a[i] != b[i] { - return false + return out +} + +type LinesTest struct { + a string + b []string +} + +var linesTests = []LinesTest{ + {a: "abc\nabc\n", b: []string{"abc\n", "abc\n"}}, + {a: "abc\r\nabc", b: []string{"abc\r\n", "abc"}}, + {a: "abc\r\n", b: []string{"abc\r\n"}}, + {a: "\nabc", b: []string{"\n", "abc"}}, + {a: "\nabc\n\n", b: []string{"\n", "abc\n", "\n"}}, +} + +func TestLines(t *testing.T) { + for _, s := range linesTests { + result := slices.Collect(Lines(s.a)) + if !slices.Equal(result, s.b) { + t.Errorf(`slices.Collect(Lines(%q)) = %q; want %q`, s.a, result, s.b) } } - return true } var abcd = "abcd" @@ -135,6 +155,11 @@ var indexTests = []IndexTest{ // test fallback to Rabin-Karp. {"oxoxoxoxoxoxoxoxoxoxoxoy", "oy", 22}, {"oxoxoxoxoxoxoxoxoxoxoxox", "oy", -1}, + // test fallback to IndexRune + {"oxoxoxoxoxoxoxoxoxoxox☺", "☺", 22}, + // invalid UTF-8 byte sequence (must be longer than bytealg.MaxBruteForce to + // test that we don't use IndexRune) + {"xx0123456789012345678901234567890123456789012345678901234567890120123456789012345678901234567890123456xxx\xed\x9f\xc0", "\xed\x9f\xc0", 105}, } var lastIndexTests = []IndexTest{ @@ -306,6 +331,37 @@ func TestIndexRune(t *testing.T) { {"a☺b☻c☹d\xe2\x98�\xff�\xed\xa0\x80", -1, -1}, {"a☺b☻c☹d\xe2\x98�\xff�\xed\xa0\x80", 0xD800, -1}, // Surrogate pair {"a☺b☻c☹d\xe2\x98�\xff�\xed\xa0\x80", utf8.MaxRune + 1, -1}, + + // 2 bytes + {"ӆ", 'ӆ', 0}, + {"a", 'ӆ', -1}, + {" ӆ", 'ӆ', 2}, + {" a", 'ӆ', -1}, + {Repeat("ц", 64) + "ӆ", 'ӆ', 128}, // test cutover + {Repeat("Ꙁ", 64) + "Ꚁ", '䚀', -1}, // 'Ꚁ' and '䚀' share the same last two bytes + + // 3 bytes + {"Ꚁ", 'Ꚁ', 0}, + {"a", 'Ꚁ', -1}, + {" Ꚁ", 'Ꚁ', 2}, + {" a", 'Ꚁ', -1}, + {Repeat("Ꙁ", 64) + "Ꚁ", 'Ꚁ', 192}, // test cutover + {Repeat("𡋀", 64) + "𡌀", '𣌀', -1}, // '𡌀' and '𣌀' share the same last two bytes + + // 4 bytes + {"𡌀", '𡌀', 0}, + {"a", '𡌀', -1}, + {" 𡌀", '𡌀', 2}, + {" a", '𡌀', -1}, + {Repeat("𡋀", 64) + "𡌀", '𡌀', 256}, // test cutover + {Repeat("𡋀", 64), '𡌀', -1}, + + // Test the cutover to bytealg.IndexString when it is triggered in + // the middle of rune that contains consecutive runs of equal bytes. + {"aaaaaKKKK\U000bc104", '\U000bc104', 17}, // cutover: (n + 16) / 8 + {"aaaaaKKKK鄄", '鄄', 17}, + {"aaKKKKKa\U000bc104", '\U000bc104', 18}, // cutover: 4 + n>>4 + {"aaKKKKKa鄄", '鄄', 18}, } for _, tt := range tests { if got := IndexRune(tt.in, tt.rune); got != tt.want { @@ -313,13 +369,14 @@ func TestIndexRune(t *testing.T) { } } - haystack := "test世界" + // Make sure we trigger the cutover and string(rune) conversion. + haystack := "test" + Repeat("𡋀", 32) + "𡌀" allocs := testing.AllocsPerRun(1000, func() { if i := IndexRune(haystack, 's'); i != 2 { t.Fatalf("'s' at %d; want 2", i) } - if i := IndexRune(haystack, '世'); i != 4 { - t.Fatalf("'世' at %d; want 4", i) + if i := IndexRune(haystack, '𡌀'); i != 132 { + t.Fatalf("'𡌀' at %d; want 4", i) } }) if allocs != 0 && testing.CoverMode() == "" { @@ -418,10 +475,16 @@ var splittests = []SplitTest{ func TestSplit(t *testing.T) { for _, tt := range splittests { a := SplitN(tt.s, tt.sep, tt.n) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("Split(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, a, tt.a) continue } + if tt.n < 0 { + a2 := slices.Collect(SplitSeq(tt.s, tt.sep)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(SplitSeq(%q, %q)) = %v; want %v`, tt.s, tt.sep, a2, tt.a) + } + } if tt.n == 0 { continue } @@ -431,7 +494,7 @@ func TestSplit(t *testing.T) { } if tt.n < 0 { b := Split(tt.s, tt.sep) - if !reflect.DeepEqual(a, b) { + if !slices.Equal(a, b) { t.Errorf("Split disagrees with SplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) } } @@ -457,17 +520,23 @@ var splitaftertests = []SplitTest{ func TestSplitAfter(t *testing.T) { for _, tt := range splitaftertests { a := SplitAfterN(tt.s, tt.sep, tt.n) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, a, tt.a) continue } + if tt.n < 0 { + a2 := slices.Collect(SplitAfterSeq(tt.s, tt.sep)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(SplitAfterSeq(%q, %q)) = %v; want %v`, tt.s, tt.sep, a2, tt.a) + } + } s := Join(a, "") if s != tt.s { t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) } if tt.n < 0 { b := SplitAfter(tt.s, tt.sep) - if !reflect.DeepEqual(a, b) { + if !slices.Equal(a, b) { t.Errorf("SplitAfter disagrees with SplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) } } @@ -500,10 +569,14 @@ var fieldstests = []FieldsTest{ func TestFields(t *testing.T) { for _, tt := range fieldstests { a := Fields(tt.s) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) continue } + a2 := collect(t, FieldsSeq(tt.s)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(FieldsSeq(%q)) = %v; want %v`, tt.s, a2, tt.a) + } } } @@ -517,7 +590,7 @@ var FieldsFuncTests = []FieldsTest{ func TestFieldsFunc(t *testing.T) { for _, tt := range fieldstests { a := FieldsFunc(tt.s, unicode.IsSpace) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, a, tt.a) continue } @@ -525,9 +598,13 @@ func TestFieldsFunc(t *testing.T) { pred := func(c rune) bool { return c == 'X' } for _, tt := range FieldsFuncTests { a := FieldsFunc(tt.s, pred) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) } + a2 := collect(t, FieldsFuncSeq(tt.s, pred)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(FieldsFuncSeq(%q)) = %v; want %v`, tt.s, a2, tt.a) + } } } diff --git a/src/sync/atomic/doc.go b/src/sync/atomic/doc.go index 7f9d64b7..4e934047 100644 --- a/src/sync/atomic/doc.go +++ b/src/sync/atomic/doc.go @@ -60,29 +60,26 @@ import ( // for 64-bit alignment of 64-bit words accessed atomically via the primitive // atomic functions (types [Int64] and [Uint64] are automatically aligned). // The first word in an allocated struct, array, or slice; in a global -// variable; or in a local variable (because the subject of all atomic operations -// will escape to the heap) can be relied upon to be 64-bit aligned. +// variable; or in a local variable (because on 32-bit architectures, the +// subject of 64-bit atomic operations will escape to the heap) can be +// relied upon to be 64-bit aligned. // SwapInt32 atomically stores new into *addr and returns the previous *addr value. // Consider using the more ergonomic and less error-prone [Int32.Swap] instead. +// +//go:noescape func SwapInt32(addr *int32, new int32) (old int32) -// SwapInt64 atomically stores new into *addr and returns the previous *addr value. -// Consider using the more ergonomic and less error-prone [Int64.Swap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func SwapInt64(addr *int64, new int64) (old int64) - // SwapUint32 atomically stores new into *addr and returns the previous *addr value. // Consider using the more ergonomic and less error-prone [Uint32.Swap] instead. +// +//go:noescape func SwapUint32(addr *uint32, new uint32) (old uint32) -// SwapUint64 atomically stores new into *addr and returns the previous *addr value. -// Consider using the more ergonomic and less error-prone [Uint64.Swap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func SwapUint64(addr *uint64, new uint64) (old uint64) - // SwapUintptr atomically stores new into *addr and returns the previous *addr value. // Consider using the more ergonomic and less error-prone [Uintptr.Swap] instead. +// +//go:noescape func SwapUintptr(addr *uintptr, new uintptr) (old uintptr) // SwapPointer atomically stores new into *addr and returns the previous *addr value. @@ -91,24 +88,20 @@ func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer) // CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value. // Consider using the more ergonomic and less error-prone [Int32.CompareAndSwap] instead. +// +//go:noescape func CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool) -// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. -// Consider using the more ergonomic and less error-prone [Int64.CompareAndSwap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool) - // CompareAndSwapUint32 executes the compare-and-swap operation for a uint32 value. // Consider using the more ergonomic and less error-prone [Uint32.CompareAndSwap] instead. +// +//go:noescape func CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool) -// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. -// Consider using the more ergonomic and less error-prone [Uint64.CompareAndSwap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool) - // CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value. // Consider using the more ergonomic and less error-prone [Uintptr.CompareAndSwap] instead. +// +//go:noescape func CompareAndSwapUintptr(addr *uintptr, old, new uintptr) (swapped bool) // CompareAndSwapPointer executes the compare-and-swap operation for a unsafe.Pointer value. @@ -117,100 +110,82 @@ func CompareAndSwapPointer(addr *unsafe.Pointer, old, new unsafe.Pointer) (swapp // AddInt32 atomically adds delta to *addr and returns the new value. // Consider using the more ergonomic and less error-prone [Int32.Add] instead. +// +//go:noescape func AddInt32(addr *int32, delta int32) (new int32) // AddUint32 atomically adds delta to *addr and returns the new value. // To subtract a signed positive constant value c from x, do AddUint32(&x, ^uint32(c-1)). // In particular, to decrement x, do AddUint32(&x, ^uint32(0)). // Consider using the more ergonomic and less error-prone [Uint32.Add] instead. +// +//go:noescape func AddUint32(addr *uint32, delta uint32) (new uint32) -// AddInt64 atomically adds delta to *addr and returns the new value. -// Consider using the more ergonomic and less error-prone [Int64.Add] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func AddInt64(addr *int64, delta int64) (new int64) - -// AddUint64 atomically adds delta to *addr and returns the new value. -// To subtract a signed positive constant value c from x, do AddUint64(&x, ^uint64(c-1)). -// In particular, to decrement x, do AddUint64(&x, ^uint64(0)). -// Consider using the more ergonomic and less error-prone [Uint64.Add] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func AddUint64(addr *uint64, delta uint64) (new uint64) - // AddUintptr atomically adds delta to *addr and returns the new value. // Consider using the more ergonomic and less error-prone [Uintptr.Add] instead. +// +//go:noescape func AddUintptr(addr *uintptr, delta uintptr) (new uintptr) // AndInt32 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Int32.And] instead. +// +//go:noescape func AndInt32(addr *int32, mask int32) (old int32) // AndUint32 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uint32.And] instead. +// +//go:noescape func AndUint32(addr *uint32, mask uint32) (old uint32) -// AndInt64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask -// and returns the old value. -// Consider using the more ergonomic and less error-prone [Int64.And] instead. -func AndInt64(addr *int64, mask int64) (old int64) - -// AndUint64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask -// and returns the old. -// Consider using the more ergonomic and less error-prone [Uint64.And] instead. -func AndUint64(addr *uint64, mask uint64) (old uint64) - // AndUintptr atomically performs a bitwise AND operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uintptr.And] instead. +// +//go:noescape func AndUintptr(addr *uintptr, mask uintptr) (old uintptr) // OrInt32 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Int32.Or] instead. +// +//go:noescape func OrInt32(addr *int32, mask int32) (old int32) // OrUint32 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uint32.Or] instead. +// +//go:noescape func OrUint32(addr *uint32, mask uint32) (old uint32) -// OrInt64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask -// and returns the old value. -// Consider using the more ergonomic and less error-prone [Int64.Or] instead. -func OrInt64(addr *int64, mask int64) (old int64) - -// OrUint64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask -// and returns the old value. -// Consider using the more ergonomic and less error-prone [Uint64.Or] instead. -func OrUint64(addr *uint64, mask uint64) (old uint64) - // OrUintptr atomically performs a bitwise OR operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uintptr.Or] instead. +// +//go:noescape func OrUintptr(addr *uintptr, mask uintptr) (old uintptr) // LoadInt32 atomically loads *addr. // Consider using the more ergonomic and less error-prone [Int32.Load] instead. +// +//go:noescape func LoadInt32(addr *int32) (val int32) -// LoadInt64 atomically loads *addr. -// Consider using the more ergonomic and less error-prone [Int64.Load] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func LoadInt64(addr *int64) (val int64) - // LoadUint32 atomically loads *addr. // Consider using the more ergonomic and less error-prone [Uint32.Load] instead. +// +//go:noescape func LoadUint32(addr *uint32) (val uint32) -// LoadUint64 atomically loads *addr. -// Consider using the more ergonomic and less error-prone [Uint64.Load] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func LoadUint64(addr *uint64) (val uint64) - // LoadUintptr atomically loads *addr. // Consider using the more ergonomic and less error-prone [Uintptr.Load] instead. +// +//go:noescape func LoadUintptr(addr *uintptr) (val uintptr) // LoadPointer atomically loads *addr. @@ -219,24 +194,20 @@ func LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer) // StoreInt32 atomically stores val into *addr. // Consider using the more ergonomic and less error-prone [Int32.Store] instead. +// +//go:noescape func StoreInt32(addr *int32, val int32) -// StoreInt64 atomically stores val into *addr. -// Consider using the more ergonomic and less error-prone [Int64.Store] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func StoreInt64(addr *int64, val int64) - // StoreUint32 atomically stores val into *addr. // Consider using the more ergonomic and less error-prone [Uint32.Store] instead. +// +//go:noescape func StoreUint32(addr *uint32, val uint32) -// StoreUint64 atomically stores val into *addr. -// Consider using the more ergonomic and less error-prone [Uint64.Store] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func StoreUint64(addr *uint64, val uint64) - // StoreUintptr atomically stores val into *addr. // Consider using the more ergonomic and less error-prone [Uintptr.Store] instead. +// +//go:noescape func StoreUintptr(addr *uintptr, val uintptr) // StorePointer atomically stores val into *addr. diff --git a/src/sync/atomic/doc_32.go b/src/sync/atomic/doc_32.go new file mode 100644 index 00000000..9d644f25 --- /dev/null +++ b/src/sync/atomic/doc_32.go @@ -0,0 +1,79 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || arm || mips || mipsle + +package atomic + +// SwapInt64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Int64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func SwapInt64(addr *int64, new int64) (old int64) + +// SwapUint64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Uint64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func SwapUint64(addr *uint64, new uint64) (old uint64) + +// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. +// Consider using the more ergonomic and less error-prone [Int64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool) + +// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. +// Consider using the more ergonomic and less error-prone [Uint64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool) + +// AddInt64 atomically adds delta to *addr and returns the new value. +// Consider using the more ergonomic and less error-prone [Int64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func AddInt64(addr *int64, delta int64) (new int64) + +// AddUint64 atomically adds delta to *addr and returns the new value. +// To subtract a signed positive constant value c from x, do AddUint64(&x, ^uint64(c-1)). +// In particular, to decrement x, do AddUint64(&x, ^uint64(0)). +// Consider using the more ergonomic and less error-prone [Uint64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func AddUint64(addr *uint64, delta uint64) (new uint64) + +// AndInt64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.And] instead. +func AndInt64(addr *int64, mask int64) (old int64) + +// AndUint64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old. +// Consider using the more ergonomic and less error-prone [Uint64.And] instead. +func AndUint64(addr *uint64, mask uint64) (old uint64) + +// OrInt64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.Or] instead. +func OrInt64(addr *int64, mask int64) (old int64) + +// OrUint64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Uint64.Or] instead. +func OrUint64(addr *uint64, mask uint64) (old uint64) + +// LoadInt64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Int64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func LoadInt64(addr *int64) (val int64) + +// LoadUint64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func LoadUint64(addr *uint64) (val uint64) + +// StoreInt64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Int64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func StoreInt64(addr *int64, val int64) + +// StoreUint64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func StoreUint64(addr *uint64, val uint64) diff --git a/src/sync/atomic/doc_64.go b/src/sync/atomic/doc_64.go new file mode 100644 index 00000000..5fec3f4e --- /dev/null +++ b/src/sync/atomic/doc_64.go @@ -0,0 +1,107 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !(386 || arm || mips || mipsle) + +package atomic + +// SwapInt64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Int64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func SwapInt64(addr *int64, new int64) (old int64) + +// SwapUint64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Uint64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func SwapUint64(addr *uint64, new uint64) (old uint64) + +// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. +// Consider using the more ergonomic and less error-prone [Int64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool) + +// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. +// Consider using the more ergonomic and less error-prone [Uint64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool) + +// AddInt64 atomically adds delta to *addr and returns the new value. +// Consider using the more ergonomic and less error-prone [Int64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func AddInt64(addr *int64, delta int64) (new int64) + +// AddUint64 atomically adds delta to *addr and returns the new value. +// To subtract a signed positive constant value c from x, do AddUint64(&x, ^uint64(c-1)). +// In particular, to decrement x, do AddUint64(&x, ^uint64(0)). +// Consider using the more ergonomic and less error-prone [Uint64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func AddUint64(addr *uint64, delta uint64) (new uint64) + +// AndInt64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.And] instead. +// +//go:noescape +func AndInt64(addr *int64, mask int64) (old int64) + +// AndUint64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old. +// Consider using the more ergonomic and less error-prone [Uint64.And] instead. +// +//go:noescape +func AndUint64(addr *uint64, mask uint64) (old uint64) + +// OrInt64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.Or] instead. +// +//go:noescape +func OrInt64(addr *int64, mask int64) (old int64) + +// OrUint64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Uint64.Or] instead. +// +//go:noescape +func OrUint64(addr *uint64, mask uint64) (old uint64) + +// LoadInt64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Int64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func LoadInt64(addr *int64) (val int64) + +// LoadUint64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func LoadUint64(addr *uint64) (val uint64) + +// StoreInt64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Int64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func StoreInt64(addr *int64, val int64) + +// StoreUint64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func StoreUint64(addr *uint64, val uint64) diff --git a/src/sync/hashtriemap.go b/src/sync/hashtriemap.go new file mode 100644 index 00000000..8df0e2b5 --- /dev/null +++ b/src/sync/hashtriemap.go @@ -0,0 +1,116 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.synchashtriemap + +package sync + +import ( + isync "internal/sync" +) + +// Map is like a Go map[any]any but is safe for concurrent use +// by multiple goroutines without additional locking or coordination. +// Loads, stores, and deletes run in amortized constant time. +// +// The Map type is specialized. Most code should use a plain Go map instead, +// with separate locking or coordination, for better type safety and to make it +// easier to maintain other invariants along with the map content. +// +// The Map type is optimized for two common use cases: (1) when the entry for a given +// key is only ever written once but read many times, as in caches that only grow, +// or (2) when multiple goroutines read, write, and overwrite entries for disjoint +// sets of keys. In these two cases, use of a Map may significantly reduce lock +// contention compared to a Go map paired with a separate [Mutex] or [RWMutex]. +// +// The zero Map is empty and ready for use. A Map must not be copied after first use. +// +// In the terminology of [the Go memory model], Map arranges that a write operation +// “synchronizes before” any read operation that observes the effect of the write, where +// read and write operations are defined as follows. +// [Map.Load], [Map.LoadAndDelete], [Map.LoadOrStore], [Map.Swap], [Map.CompareAndSwap], +// and [Map.CompareAndDelete] are read operations; +// [Map.Delete], [Map.LoadAndDelete], [Map.Store], and [Map.Swap] are write operations; +// [Map.LoadOrStore] is a write operation when it returns loaded set to false; +// [Map.CompareAndSwap] is a write operation when it returns swapped set to true; +// and [Map.CompareAndDelete] is a write operation when it returns deleted set to true. +// +// [the Go memory model]: https://go.dev/ref/mem +type Map struct { + _ noCopy + + m isync.HashTrieMap[any, any] +} + +// Load returns the value stored in the map for a key, or nil if no +// value is present. +// The ok result indicates whether value was found in the map. +func (m *Map) Load(key any) (value any, ok bool) { + return m.m.Load(key) +} + +// Store sets the value for a key. +func (m *Map) Store(key, value any) { + m.m.Store(key, value) +} + +// Clear deletes all the entries, resulting in an empty Map. +func (m *Map) Clear() { + m.m.Clear() +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *Map) LoadOrStore(key, value any) (actual any, loaded bool) { + return m.m.LoadOrStore(key, value) +} + +// LoadAndDelete deletes the value for a key, returning the previous value if any. +// The loaded result reports whether the key was present. +func (m *Map) LoadAndDelete(key any) (value any, loaded bool) { + return m.m.LoadAndDelete(key) +} + +// Delete deletes the value for a key. +func (m *Map) Delete(key any) { + m.m.Delete(key) +} + +// Swap swaps the value for a key and returns the previous value if any. +// The loaded result reports whether the key was present. +func (m *Map) Swap(key, value any) (previous any, loaded bool) { + return m.m.Swap(key, value) +} + +// CompareAndSwap swaps the old and new values for key +// if the value stored in the map is equal to old. +// The old value must be of a comparable type. +func (m *Map) CompareAndSwap(key, old, new any) (swapped bool) { + return m.m.CompareAndSwap(key, old, new) +} + +// CompareAndDelete deletes the entry for key if its value is equal to old. +// The old value must be of a comparable type. +// +// If there is no current value for key in the map, CompareAndDelete +// returns false (even if the old value is the nil interface value). +func (m *Map) CompareAndDelete(key, old any) (deleted bool) { + return m.m.CompareAndDelete(key, old) +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// Range does not necessarily correspond to any consistent snapshot of the Map's +// contents: no key will be visited more than once, but if the value for any key +// is stored or deleted concurrently (including by f), Range may reflect any +// mapping for that key from any point during the Range call. Range does not +// block other methods on the receiver; even f itself may call any method on m. +// +// Range may be O(N) with the number of elements in the map even if f returns +// false after a constant number of calls. +func (m *Map) Range(f func(key, value any) bool) { + m.m.Range(f) +} diff --git a/src/sync/map.go b/src/sync/map.go index 33bc8141..25181e0c 100644 --- a/src/sync/map.go +++ b/src/sync/map.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.synchashtriemap + package sync import ( @@ -36,6 +38,8 @@ import ( // // [the Go memory model]: https://go.dev/ref/mem type Map struct { + _ noCopy + mu Mutex // read contains the portion of the map's contents that are safe for diff --git a/src/sync/map_bench_test.go b/src/sync/map_bench_test.go index fb9eb254..f7469aed 100644 --- a/src/sync/map_bench_test.go +++ b/src/sync/map_bench_test.go @@ -6,6 +6,7 @@ package sync_test import ( "fmt" + isync "internal/sync" "reflect" "sync" "sync/atomic" @@ -18,13 +19,14 @@ type bench struct { } func benchMap(b *testing.B, bench bench) { - for _, m := range [...]mapInterface{&DeepCopyMap{}, &RWMutexMap{}, &sync.Map{}} { + for _, m := range [...]mapInterface{&DeepCopyMap{}, &RWMutexMap{}, &isync.HashTrieMap[any, any]{}, &sync.Map{}} { b.Run(fmt.Sprintf("%T", m), func(b *testing.B) { m = reflect.New(reflect.TypeOf(m).Elem()).Interface().(mapInterface) if bench.setup != nil { bench.setup(b, m) } + b.ReportAllocs() b.ResetTimer() var i int64 @@ -36,7 +38,7 @@ func benchMap(b *testing.B, bench bench) { } } -func BenchmarkLoadMostlyHits(b *testing.B) { +func BenchmarkMapLoadMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -58,7 +60,7 @@ func BenchmarkLoadMostlyHits(b *testing.B) { }) } -func BenchmarkLoadMostlyMisses(b *testing.B) { +func BenchmarkMapLoadMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -80,7 +82,7 @@ func BenchmarkLoadMostlyMisses(b *testing.B) { }) } -func BenchmarkLoadOrStoreBalanced(b *testing.B) { +func BenchmarkMapLoadOrStoreBalanced(b *testing.B) { const hits, misses = 128, 128 benchMap(b, bench{ @@ -114,7 +116,7 @@ func BenchmarkLoadOrStoreBalanced(b *testing.B) { }) } -func BenchmarkLoadOrStoreUnique(b *testing.B) { +func BenchmarkMapLoadOrStoreUnique(b *testing.B) { benchMap(b, bench{ setup: func(b *testing.B, m mapInterface) { if _, ok := m.(*DeepCopyMap); ok { @@ -130,7 +132,7 @@ func BenchmarkLoadOrStoreUnique(b *testing.B) { }) } -func BenchmarkLoadOrStoreCollision(b *testing.B) { +func BenchmarkMapLoadOrStoreCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -144,7 +146,7 @@ func BenchmarkLoadOrStoreCollision(b *testing.B) { }) } -func BenchmarkLoadAndDeleteBalanced(b *testing.B) { +func BenchmarkMapLoadAndDeleteBalanced(b *testing.B) { const hits, misses = 128, 128 benchMap(b, bench{ @@ -174,7 +176,7 @@ func BenchmarkLoadAndDeleteBalanced(b *testing.B) { }) } -func BenchmarkLoadAndDeleteUnique(b *testing.B) { +func BenchmarkMapLoadAndDeleteUnique(b *testing.B) { benchMap(b, bench{ setup: func(b *testing.B, m mapInterface) { if _, ok := m.(*DeepCopyMap); ok { @@ -190,7 +192,7 @@ func BenchmarkLoadAndDeleteUnique(b *testing.B) { }) } -func BenchmarkLoadAndDeleteCollision(b *testing.B) { +func BenchmarkMapLoadAndDeleteCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -206,7 +208,7 @@ func BenchmarkLoadAndDeleteCollision(b *testing.B) { }) } -func BenchmarkRange(b *testing.B) { +func BenchmarkMapRange(b *testing.B) { const mapSize = 1 << 10 benchMap(b, bench{ @@ -224,12 +226,12 @@ func BenchmarkRange(b *testing.B) { }) } -// BenchmarkAdversarialAlloc tests performance when we store a new value +// BenchmarkMapAdversarialAlloc tests performance when we store a new value // immediately whenever the map is promoted to clean and otherwise load a // unique, missing key. // // This forces the Load calls to always acquire the map's mutex. -func BenchmarkAdversarialAlloc(b *testing.B) { +func BenchmarkMapAdversarialAlloc(b *testing.B) { benchMap(b, bench{ perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { var stores, loadsSinceStore int64 @@ -245,12 +247,12 @@ func BenchmarkAdversarialAlloc(b *testing.B) { }) } -// BenchmarkAdversarialDelete tests performance when we periodically delete +// BenchmarkMapAdversarialDelete tests performance when we periodically delete // one key and add a different one in a large map. // // This forces the Load calls to always acquire the map's mutex and periodically // makes a full copy of the map despite changing only one entry. -func BenchmarkAdversarialDelete(b *testing.B) { +func BenchmarkMapAdversarialDelete(b *testing.B) { const mapSize = 1 << 10 benchMap(b, bench{ @@ -276,7 +278,7 @@ func BenchmarkAdversarialDelete(b *testing.B) { }) } -func BenchmarkDeleteCollision(b *testing.B) { +func BenchmarkMapDeleteCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -290,7 +292,7 @@ func BenchmarkDeleteCollision(b *testing.B) { }) } -func BenchmarkSwapCollision(b *testing.B) { +func BenchmarkMapSwapCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -304,7 +306,7 @@ func BenchmarkSwapCollision(b *testing.B) { }) } -func BenchmarkSwapMostlyHits(b *testing.B) { +func BenchmarkMapSwapMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -332,7 +334,7 @@ func BenchmarkSwapMostlyHits(b *testing.B) { }) } -func BenchmarkSwapMostlyMisses(b *testing.B) { +func BenchmarkMapSwapMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -360,7 +362,7 @@ func BenchmarkSwapMostlyMisses(b *testing.B) { }) } -func BenchmarkCompareAndSwapCollision(b *testing.B) { +func BenchmarkMapCompareAndSwapCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -376,7 +378,7 @@ func BenchmarkCompareAndSwapCollision(b *testing.B) { }) } -func BenchmarkCompareAndSwapNoExistingKey(b *testing.B) { +func BenchmarkMapCompareAndSwapNoExistingKey(b *testing.B) { benchMap(b, bench{ perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { for ; pb.Next(); i++ { @@ -388,7 +390,7 @@ func BenchmarkCompareAndSwapNoExistingKey(b *testing.B) { }) } -func BenchmarkCompareAndSwapValueNotEqual(b *testing.B) { +func BenchmarkMapCompareAndSwapValueNotEqual(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.Store(0, 0) @@ -402,7 +404,7 @@ func BenchmarkCompareAndSwapValueNotEqual(b *testing.B) { }) } -func BenchmarkCompareAndSwapMostlyHits(b *testing.B) { +func BenchmarkMapCompareAndSwapMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -432,7 +434,7 @@ func BenchmarkCompareAndSwapMostlyHits(b *testing.B) { }) } -func BenchmarkCompareAndSwapMostlyMisses(b *testing.B) { +func BenchmarkMapCompareAndSwapMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -458,7 +460,7 @@ func BenchmarkCompareAndSwapMostlyMisses(b *testing.B) { }) } -func BenchmarkCompareAndDeleteCollision(b *testing.B) { +func BenchmarkMapCompareAndDeleteCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -474,7 +476,7 @@ func BenchmarkCompareAndDeleteCollision(b *testing.B) { }) } -func BenchmarkCompareAndDeleteMostlyHits(b *testing.B) { +func BenchmarkMapCompareAndDeleteMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -506,7 +508,7 @@ func BenchmarkCompareAndDeleteMostlyHits(b *testing.B) { }) } -func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { +func BenchmarkMapCompareAndDeleteMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -534,7 +536,7 @@ func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { }) } -func BenchmarkClear(b *testing.B) { +func BenchmarkMapClear(b *testing.B) { benchMap(b, bench{ perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { for ; pb.Next(); i++ { diff --git a/src/sync/map_reference_test.go b/src/sync/map_reference_test.go index 283da0f3..f98bb98b 100644 --- a/src/sync/map_reference_test.go +++ b/src/sync/map_reference_test.go @@ -5,6 +5,7 @@ package sync_test import ( + isync "internal/sync" "sync" "sync/atomic" ) @@ -28,6 +29,7 @@ type mapInterface interface { var ( _ mapInterface = &RWMutexMap{} _ mapInterface = &DeepCopyMap{} + _ mapInterface = &isync.HashTrieMap[any, any]{} ) // RWMutexMap is an implementation of mapInterface using a sync.RWMutex. diff --git a/src/sync/map_test.go b/src/sync/map_test.go index e1d03807..f12c43a2 100644 --- a/src/sync/map_test.go +++ b/src/sync/map_test.go @@ -5,6 +5,7 @@ package sync_test import ( + isync "internal/sync" "internal/testenv" "math/rand" "reflect" @@ -133,6 +134,10 @@ func applyDeepCopyMap(calls []mapCall) ([]mapResult, map[any]any) { return applyCalls(new(DeepCopyMap), calls) } +func applyHashTrieMap(calls []mapCall) ([]mapResult, map[any]any) { + return applyCalls(new(isync.HashTrieMap[any, any]), calls) +} + func TestMapMatchesRWMutex(t *testing.T) { if err := quick.CheckEqual(applyMap, applyRWMutexMap, nil); err != nil { t.Error(err) @@ -145,6 +150,12 @@ func TestMapMatchesDeepCopy(t *testing.T) { } } +func TestMapMatchesHashTrieMap(t *testing.T) { + if err := quick.CheckEqual(applyMap, applyHashTrieMap, nil); err != nil { + t.Error(err) + } +} + func TestConcurrentRange(t *testing.T) { const mapSize = 1 << 10 @@ -347,13 +358,13 @@ func TestConcurrentClear(t *testing.T) { }) } -func TestMapClearNoAllocations(t *testing.T) { +func TestMapClearOneAllocation(t *testing.T) { testenv.SkipIfOptimizationOff(t) var m sync.Map allocs := testing.AllocsPerRun(10, func() { m.Clear() }) - if allocs > 0 { - t.Errorf("AllocsPerRun of m.Clear = %v; want 0", allocs) + if allocs > 1 { + t.Errorf("AllocsPerRun of m.Clear = %v; want 1", allocs) } } diff --git a/src/sync/mutex.go b/src/sync/mutex.go index e4ed47c7..133c9530 100644 --- a/src/sync/mutex.go +++ b/src/sync/mutex.go @@ -11,15 +11,9 @@ package sync import ( - "internal/race" - "sync/atomic" - "unsafe" + isync "internal/sync" ) -// Provided by runtime via linkname. -func throw(string) -func fatal(string) - // A Mutex is a mutual exclusion lock. // The zero value for a Mutex is an unlocked mutex. // @@ -34,8 +28,9 @@ func fatal(string) // // [the Go memory model]: https://go.dev/ref/mem type Mutex struct { - state int32 - sema uint32 + _ noCopy + + mu isync.Mutex } // A Locker represents an object that can be locked and unlocked. @@ -44,52 +39,11 @@ type Locker interface { Unlock() } -const ( - mutexLocked = 1 << iota // mutex is locked - mutexWoken - mutexStarving - mutexWaiterShift = iota - - // Mutex fairness. - // - // Mutex can be in 2 modes of operations: normal and starvation. - // In normal mode waiters are queued in FIFO order, but a woken up waiter - // does not own the mutex and competes with new arriving goroutines over - // the ownership. New arriving goroutines have an advantage -- they are - // already running on CPU and there can be lots of them, so a woken up - // waiter has good chances of losing. In such case it is queued at front - // of the wait queue. If a waiter fails to acquire the mutex for more than 1ms, - // it switches mutex to the starvation mode. - // - // In starvation mode ownership of the mutex is directly handed off from - // the unlocking goroutine to the waiter at the front of the queue. - // New arriving goroutines don't try to acquire the mutex even if it appears - // to be unlocked, and don't try to spin. Instead they queue themselves at - // the tail of the wait queue. - // - // If a waiter receives ownership of the mutex and sees that either - // (1) it is the last waiter in the queue, or (2) it waited for less than 1 ms, - // it switches mutex back to normal operation mode. - // - // Normal mode has considerably better performance as a goroutine can acquire - // a mutex several times in a row even if there are blocked waiters. - // Starvation mode is important to prevent pathological cases of tail latency. - starvationThresholdNs = 1e6 -) - // Lock locks m. // If the lock is already in use, the calling goroutine // blocks until the mutex is available. func (m *Mutex) Lock() { - // Fast path: grab unlocked mutex. - if atomic.CompareAndSwapInt32(&m.state, 0, mutexLocked) { - if race.Enabled { - race.Acquire(unsafe.Pointer(m)) - } - return - } - // Slow path (outlined so that the fast path can be inlined) - m.lockSlow() + m.mu.Lock() } // TryLock tries to lock m and reports whether it succeeded. @@ -98,111 +52,7 @@ func (m *Mutex) Lock() { // and use of TryLock is often a sign of a deeper problem // in a particular use of mutexes. func (m *Mutex) TryLock() bool { - old := m.state - if old&(mutexLocked|mutexStarving) != 0 { - return false - } - - // There may be a goroutine waiting for the mutex, but we are - // running now and can try to grab the mutex before that - // goroutine wakes up. - if !atomic.CompareAndSwapInt32(&m.state, old, old|mutexLocked) { - return false - } - - if race.Enabled { - race.Acquire(unsafe.Pointer(m)) - } - return true -} - -func (m *Mutex) lockSlow() { - var waitStartTime int64 - starving := false - awoke := false - iter := 0 - old := m.state - for { - // Don't spin in starvation mode, ownership is handed off to waiters - // so we won't be able to acquire the mutex anyway. - if old&(mutexLocked|mutexStarving) == mutexLocked && runtime_canSpin(iter) { - // Active spinning makes sense. - // Try to set mutexWoken flag to inform Unlock - // to not wake other blocked goroutines. - if !awoke && old&mutexWoken == 0 && old>>mutexWaiterShift != 0 && - atomic.CompareAndSwapInt32(&m.state, old, old|mutexWoken) { - awoke = true - } - runtime_doSpin() - iter++ - old = m.state - continue - } - new := old - // Don't try to acquire starving mutex, new arriving goroutines must queue. - if old&mutexStarving == 0 { - new |= mutexLocked - } - if old&(mutexLocked|mutexStarving) != 0 { - new += 1 << mutexWaiterShift - } - // The current goroutine switches mutex to starvation mode. - // But if the mutex is currently unlocked, don't do the switch. - // Unlock expects that starving mutex has waiters, which will not - // be true in this case. - if starving && old&mutexLocked != 0 { - new |= mutexStarving - } - if awoke { - // The goroutine has been woken from sleep, - // so we need to reset the flag in either case. - if new&mutexWoken == 0 { - throw("sync: inconsistent mutex state") - } - new &^= mutexWoken - } - if atomic.CompareAndSwapInt32(&m.state, old, new) { - if old&(mutexLocked|mutexStarving) == 0 { - break // locked the mutex with CAS - } - // If we were already waiting before, queue at the front of the queue. - queueLifo := waitStartTime != 0 - if waitStartTime == 0 { - waitStartTime = runtime_nanotime() - } - runtime_SemacquireMutex(&m.sema, queueLifo, 1) - starving = starving || runtime_nanotime()-waitStartTime > starvationThresholdNs - old = m.state - if old&mutexStarving != 0 { - // If this goroutine was woken and mutex is in starvation mode, - // ownership was handed off to us but mutex is in somewhat - // inconsistent state: mutexLocked is not set and we are still - // accounted as waiter. Fix that. - if old&(mutexLocked|mutexWoken) != 0 || old>>mutexWaiterShift == 0 { - throw("sync: inconsistent mutex state") - } - delta := int32(mutexLocked - 1<>mutexWaiterShift == 1 { - // Exit starvation mode. - // Critical to do it here and consider wait time. - // Starvation mode is so inefficient, that two goroutines - // can go lock-step infinitely once they switch mutex - // to starvation mode. - delta -= mutexStarving - } - atomic.AddInt32(&m.state, delta) - break - } - awoke = true - iter = 0 - } else { - old = m.state - } - } - - if race.Enabled { - race.Acquire(unsafe.Pointer(m)) - } + return m.mu.TryLock() } // Unlock unlocks m. @@ -212,50 +62,5 @@ func (m *Mutex) lockSlow() { // It is allowed for one goroutine to lock a Mutex and then // arrange for another goroutine to unlock it. func (m *Mutex) Unlock() { - if race.Enabled { - _ = m.state - race.Release(unsafe.Pointer(m)) - } - - // Fast path: drop lock bit. - new := atomic.AddInt32(&m.state, -mutexLocked) - if new != 0 { - // Outlined slow path to allow inlining the fast path. - // To hide unlockSlow during tracing we skip one extra frame when tracing GoUnblock. - m.unlockSlow(new) - } -} - -func (m *Mutex) unlockSlow(new int32) { - if (new+mutexLocked)&mutexLocked == 0 { - fatal("sync: unlock of unlocked mutex") - } - if new&mutexStarving == 0 { - old := new - for { - // If there are no waiters or a goroutine has already - // been woken or grabbed the lock, no need to wake anyone. - // In starvation mode ownership is directly handed off from unlocking - // goroutine to the next waiter. We are not part of this chain, - // since we did not observe mutexStarving when we unlocked the mutex above. - // So get off the way. - if old>>mutexWaiterShift == 0 || old&(mutexLocked|mutexWoken|mutexStarving) != 0 { - return - } - // Grab the right to wake someone. - new = (old - 1< 0 { + growStack(n - 1) + } +} + +func TestWasmExport(t *testing.T) { + testExportCalled = false + a := int32(123) + b := int64(456) + want := int64(a) + b + if got := testCallExport(a, b); got != want { + t.Errorf("got %v, want %v", got, want) + } + if !testExportCalled { + t.Error("testExport not called") + } +} + func TestBool(t *testing.T) { want := true o := dummys.Get("someBool") @@ -587,16 +632,16 @@ func TestGarbageCollection(t *testing.T) { // Note: All JavaScript functions return a JavaScript array, which will cause // one allocation to be created to track the Value.gcPtr for the Value finalizer. var allocTests = []struct { - argLen int // The number of arguments to use for the syscall + argLen int // The number of arguments to use for the syscall expected int // The expected number of allocations }{ - // For less than or equal to 16 arguments, we expect 1 alloction: + // For less than or equal to 16 arguments, we expect 1 allocation: // - makeValue new(ref) - {0, 1}, - {2, 1}, + {0, 1}, + {2, 1}, {15, 1}, {16, 1}, - // For greater than 16 arguments, we expect 3 alloction: + // For greater than 16 arguments, we expect 3 allocation: // - makeValue: new(ref) // - makeArgSlices: argVals = make([]Value, size) // - makeArgSlices: argRefs = make([]ref, size) @@ -613,7 +658,7 @@ func TestCallAllocations(t *testing.T) { tmpArray := js.Global().Get("Array").New(0) numAllocs := testing.AllocsPerRun(100, func() { tmpArray.Call("concat", args...) - }); + }) if numAllocs != float64(test.expected) { t.Errorf("got numAllocs %#v, want %#v", numAllocs, test.expected) @@ -630,7 +675,7 @@ func TestInvokeAllocations(t *testing.T) { concatFunc := tmpArray.Get("concat").Call("bind", tmpArray) numAllocs := testing.AllocsPerRun(100, func() { concatFunc.Invoke(args...) - }); + }) if numAllocs != float64(test.expected) { t.Errorf("got numAllocs %#v, want %#v", numAllocs, test.expected) @@ -647,7 +692,7 @@ func TestNewAllocations(t *testing.T) { numAllocs := testing.AllocsPerRun(100, func() { arrayConstructor.New(args...) - }); + }) if numAllocs != float64(test.expected) { t.Errorf("got numAllocs %#v, want %#v", numAllocs, test.expected) diff --git a/src/syscall/linkname_openbsd.go b/src/syscall/linkname_openbsd.go index 5f5c517a..bbb56c10 100644 --- a/src/syscall/linkname_openbsd.go +++ b/src/syscall/linkname_openbsd.go @@ -12,4 +12,3 @@ import _ "unsafe" //go:linkname unlinkat //go:linkname openat //go:linkname fstatat -//go:linkname getentropy diff --git a/src/syscall/mkall.sh b/src/syscall/mkall.sh index a3bc7676..b9a0ed3d 100755 --- a/src/syscall/mkall.sh +++ b/src/syscall/mkall.sh @@ -190,7 +190,7 @@ linux_amd64) mktypes="GOARCH=$GOARCH go tool cgo -godefs" ;; linux_arm) - GOOSARCH_in="syscall_linux_arm.go syscall_linux_accept.go" + GOOSARCH_in="syscall_linux_arm.go" mkerrors="$mkerrors" mksyscall="./mksyscall.pl -l32 -arm" mksysnum="curl -s 'http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/plain/arch/arm/include/uapi/asm/unistd.h' | ./mksysnum_linux.pl -" diff --git a/src/syscall/net_wasip1.go b/src/syscall/net_wasip1.go index 3918840a..fefd939a 100644 --- a/src/syscall/net_wasip1.go +++ b/src/syscall/net_wasip1.go @@ -6,8 +6,6 @@ package syscall -import "unsafe" - const ( SHUT_RD = 0x1 SHUT_WR = 0x2 @@ -18,7 +16,7 @@ type sdflags = uint32 //go:wasmimport wasi_snapshot_preview1 sock_accept //go:noescape -func sock_accept(fd int32, flags fdflags, newfd unsafe.Pointer) Errno +func sock_accept(fd int32, flags fdflags, newfd *int32) Errno //go:wasmimport wasi_snapshot_preview1 sock_shutdown //go:noescape @@ -42,7 +40,7 @@ func Listen(fd int, backlog int) error { func Accept(fd int) (int, Sockaddr, error) { var newfd int32 - errno := sock_accept(int32(fd), 0, unsafe.Pointer(&newfd)) + errno := sock_accept(int32(fd), 0, &newfd) return int(newfd), nil, errnoErr(errno) } diff --git a/src/syscall/rlimit.go b/src/syscall/rlimit.go index 8184f17a..3812303f 100644 --- a/src/syscall/rlimit.go +++ b/src/syscall/rlimit.go @@ -29,10 +29,18 @@ var origRlimitNofile atomic.Pointer[Rlimit] // which Go of course has no choice but to respect. func init() { var lim Rlimit - if err := Getrlimit(RLIMIT_NOFILE, &lim); err == nil && lim.Cur != lim.Max { + if err := Getrlimit(RLIMIT_NOFILE, &lim); err == nil && lim.Max > 0 && lim.Cur < lim.Max-1 { origRlimitNofile.Store(&lim) nlim := lim - nlim.Cur = nlim.Max + + // We set Cur to Max - 1 so that we are more likely to + // detect cases where another process uses prlimit + // to change our resource limits. The theory is that + // using prlimit to change to Cur == Max is more likely + // than using prlimit to change to Cur == Max - 1. + // The place we check for this is in exec_linux.go. + nlim.Cur = nlim.Max - 1 + adjustFileLimit(&nlim) setrlimit(RLIMIT_NOFILE, &nlim) } diff --git a/src/syscall/syscall_aix.go b/src/syscall/syscall_aix.go index a9bd7a37..36dfd906 100644 --- a/src/syscall/syscall_aix.go +++ b/src/syscall/syscall_aix.go @@ -119,11 +119,11 @@ func Getwd() (ret string, err error) { b := make([]byte, len) err := getcwd(&b[0], len) if err == nil { - i := 0 - for b[i] != 0 { - i++ + n := clen(b[:]) + if n < 1 { + return "", EINVAL } - return string(b[0:i]), nil + return string(b[:n]), nil } if err != ERANGE { return "", err diff --git a/src/syscall/syscall_freebsd_386.go b/src/syscall/syscall_freebsd_386.go index 60359e38..a217dc75 100644 --- a/src/syscall/syscall_freebsd_386.go +++ b/src/syscall/syscall_freebsd_386.go @@ -36,7 +36,13 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e var writtenOut uint64 = 0 _, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr((*offset)>>32), uintptr(count), 0, uintptr(unsafe.Pointer(&writtenOut)), 0, 0) - written = int(writtenOut) + // For some reason on the freebsd-386 builder writtenOut + // is modified when the system call returns EINVAL. + // The man page says that the value is only written for + // success, EINTR, or EAGAIN, so only use those cases. + if e1 == 0 || e1 == EINTR || e1 == EAGAIN { + written = int(writtenOut) + } if e1 != 0 { err = e1 diff --git a/src/syscall/syscall_js.go b/src/syscall/syscall_js.go index 0e529e03..c320e34f 100644 --- a/src/syscall/syscall_js.go +++ b/src/syscall/syscall_js.go @@ -128,12 +128,13 @@ const ( O_WRONLY = 1 O_RDWR = 2 - O_CREAT = 0100 - O_CREATE = O_CREAT - O_TRUNC = 01000 - O_APPEND = 02000 - O_EXCL = 0200 - O_SYNC = 010000 + O_CREAT = 0100 + O_CREATE = O_CREAT + O_TRUNC = 01000 + O_APPEND = 02000 + O_EXCL = 0200 + O_SYNC = 010000 + O_DIRECTORY = 020000 O_CLOEXEC = 0 ) diff --git a/src/syscall/syscall_linux.go b/src/syscall/syscall_linux.go index 27069735..003f7a53 100644 --- a/src/syscall/syscall_linux.go +++ b/src/syscall/syscall_linux.go @@ -682,6 +682,10 @@ func anyToSockaddr(rsa *RawSockaddrAny) (Sockaddr, error) { return nil, EAFNOSUPPORT } +func Accept(fd int) (nfd int, sa Sockaddr, err error) { + return Accept4(fd, 0) +} + func Accept4(fd int, flags int) (nfd int, sa Sockaddr, err error) { var rsa RawSockaddrAny var len _Socklen = SizeofSockaddrAny @@ -1284,6 +1288,17 @@ func Munmap(b []byte) (err error) { //sys Mlockall(flags int) (err error) //sys Munlockall() (err error) +func Getrlimit(resource int, rlim *Rlimit) (err error) { + // prlimit1 is the same as prlimit when newlimit == nil + return prlimit1(0, resource, nil, rlim) +} + +// setrlimit sets a resource limit. +// The Setrlimit function is in rlimit.go, and calls this one. +func setrlimit(resource int, rlim *Rlimit) (err error) { + return prlimit1(0, resource, rlim, nil) +} + // prlimit changes a resource limit. We use a single definition so that // we can tell StartProcess to not restore the original NOFILE limit. // diff --git a/src/syscall/syscall_linux_386.go b/src/syscall/syscall_linux_386.go index a559f7e2..a90565a0 100644 --- a/src/syscall/syscall_linux_386.go +++ b/src/syscall/syscall_linux_386.go @@ -72,96 +72,6 @@ func mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int6 return mmap2(addr, length, prot, flags, fd, page) } -type rlimit32 struct { - Cur uint32 - Max uint32 -} - -//sysnb getrlimit(resource int, rlim *rlimit32) (err error) = SYS_GETRLIMIT - -const rlimInf32 = ^uint32(0) -const rlimInf64 = ^uint64(0) - -func Getrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - err = getrlimit(resource, &rl) - if err != nil { - return - } - - if rl.Cur == rlimInf32 { - rlim.Cur = rlimInf64 - } else { - rlim.Cur = uint64(rl.Cur) - } - - if rl.Max == rlimInf32 { - rlim.Max = rlimInf64 - } else { - rlim.Max = uint64(rl.Max) - } - return -} - -//sysnb setrlimit1(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func setrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit1(resource, &rl) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - // Underlying system call writes to newoffset via pointer. // Implemented in assembly to avoid allocation. func seek(fd int, offset int64, whence int) (newoffset int64, err Errno) diff --git a/src/syscall/syscall_linux_accept.go b/src/syscall/syscall_linux_accept.go deleted file mode 100644 index 66c0f84c..00000000 --- a/src/syscall/syscall_linux_accept.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// We require Linux kernel version 2.6.32. The accept4 system call was -// added in version 2.6.28, so in general we can use accept4. -// Unfortunately, for ARM only, accept4 was added in version 2.6.36. -// Handle that case here, by using a copy of the Accept function that -// we used in Go 1.17. - -//go:build linux && arm - -package syscall - -//sys accept(s int, rsa *RawSockaddrAny, addrlen *_Socklen) (fd int, err error) - -func Accept(fd int) (nfd int, sa Sockaddr, err error) { - var rsa RawSockaddrAny - var len _Socklen = SizeofSockaddrAny - // Try accept4 first for Android and newer kernels. - nfd, err = accept4(fd, &rsa, &len, 0) - if err == ENOSYS { - nfd, err = accept(fd, &rsa, &len) - } - if err != nil { - return - } - sa, err = anyToSockaddr(&rsa) - if err != nil { - Close(nfd) - nfd = 0 - } - return -} diff --git a/src/syscall/syscall_linux_accept4.go b/src/syscall/syscall_linux_accept4.go deleted file mode 100644 index 74898672..00000000 --- a/src/syscall/syscall_linux_accept4.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file provides the Accept function used on all systems -// other than arm. See syscall_linux_accept.go for why. - -//go:build linux && !arm - -package syscall - -func Accept(fd int) (nfd int, sa Sockaddr, err error) { - var rsa RawSockaddrAny - var len _Socklen = SizeofSockaddrAny - nfd, err = accept4(fd, &rsa, &len, 0) - if err != nil { - return - } - sa, err = anyToSockaddr(&rsa) - if err != nil { - Close(nfd) - nfd = 0 - } - return -} diff --git a/src/syscall/syscall_linux_amd64.go b/src/syscall/syscall_linux_amd64.go index ec52f8a4..725ebdba 100644 --- a/src/syscall/syscall_linux_amd64.go +++ b/src/syscall/syscall_linux_amd64.go @@ -4,10 +4,6 @@ package syscall -import ( - "unsafe" -) - const ( _SYS_setgroups = SYS_SETGROUPS _SYS_clone3 = 435 @@ -23,7 +19,6 @@ const ( //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Ioperm(from int, num int, on int) (err error) @@ -38,7 +33,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Statfs(path string, buf *Statfs_t) (err error) @@ -103,12 +97,6 @@ func Time(t *Time_t) (tt Time_t, err error) { //sys Utime(path string, buf *Utimbuf) (err error) //sys utimes(path string, times *[2]Timeval) (err error) -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func setTimespec(sec, nsec int64) Timespec { return Timespec{Sec: sec, Nsec: nsec} } diff --git a/src/syscall/syscall_linux_arm.go b/src/syscall/syscall_linux_arm.go index a6d92cea..2019b151 100644 --- a/src/syscall/syscall_linux_arm.go +++ b/src/syscall/syscall_linux_arm.go @@ -124,96 +124,6 @@ func mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int6 return mmap2(addr, length, prot, flags, fd, page) } -type rlimit32 struct { - Cur uint32 - Max uint32 -} - -//sysnb getrlimit(resource int, rlim *rlimit32) (err error) = SYS_GETRLIMIT - -const rlimInf32 = ^uint32(0) -const rlimInf64 = ^uint64(0) - -func Getrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - err = getrlimit(resource, &rl) - if err != nil { - return - } - - if rl.Cur == rlimInf32 { - rlim.Cur = rlimInf64 - } else { - rlim.Cur = uint64(rl.Cur) - } - - if rl.Max == rlimInf32 { - rlim.Max = rlimInf64 - } else { - rlim.Max = uint64(rl.Max) - } - return -} - -//sysnb setrlimit1(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func setrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit1(resource, &rl) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return uint64(r.Uregs[15]) } func (r *PtraceRegs) SetPC(pc uint64) { r.Uregs[15] = uint32(pc) } diff --git a/src/syscall/syscall_linux_arm64.go b/src/syscall/syscall_linux_arm64.go index b87b51c0..fb0ecd77 100644 --- a/src/syscall/syscall_linux_arm64.go +++ b/src/syscall/syscall_linux_arm64.go @@ -27,7 +27,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sys Listen(s int, n int) (err error) //sys pread(fd int, p []byte, offset int64) (n int, err error) = SYS_PREAD64 @@ -37,7 +36,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit1(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) @@ -140,34 +138,6 @@ func utimes(path string, tv *[2]Timeval) (err error) { return utimensat(_AT_FDCWD, path, (*[2]Timespec)(unsafe.Pointer(&ts[0])), 0) } -// Getrlimit prefers the prlimit64 system call. See issue 38604. -func Getrlimit(resource int, rlim *Rlimit) error { - err := prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - return getrlimit(resource, rlim) -} - -// setrlimit prefers the prlimit64 system call. See issue 38604. -func setrlimit(resource int, rlim *Rlimit) error { - err := prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - return setrlimit1(resource, rlim) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return r.Pc } func (r *PtraceRegs) SetPC(pc uint64) { r.Pc = pc } diff --git a/src/syscall/syscall_linux_loong64.go b/src/syscall/syscall_linux_loong64.go index 634cf30c..360590d2 100644 --- a/src/syscall/syscall_linux_loong64.go +++ b/src/syscall/syscall_linux_loong64.go @@ -183,22 +183,6 @@ func utimes(path string, tv *[2]Timeval) (err error) { return utimensat(_AT_FDCWD, path, (*[2]Timespec)(unsafe.Pointer(&ts[0])), 0) } -// Getrlimit prefers the prlimit64 system call. -func Getrlimit(resource int, rlim *Rlimit) error { - return prlimit(0, resource, nil, rlim) -} - -// setrlimit prefers the prlimit64 system call. -func setrlimit(resource int, rlim *Rlimit) error { - return prlimit(0, resource, rlim, nil) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - return errno -} - func (r *PtraceRegs) GetEra() uint64 { return r.Era } func (r *PtraceRegs) SetEra(era uint64) { r.Era = era } diff --git a/src/syscall/syscall_linux_mips64x.go b/src/syscall/syscall_linux_mips64x.go index 41106ed8..e826e086 100644 --- a/src/syscall/syscall_linux_mips64x.go +++ b/src/syscall/syscall_linux_mips64x.go @@ -6,10 +6,6 @@ package syscall -import ( - "unsafe" -) - const ( _SYS_setgroups = SYS_SETGROUPS _SYS_clone3 = 5435 @@ -20,12 +16,10 @@ const ( //sys Dup2(oldfd int, newfd int) (err error) //sys Fchown(fd int, uid int, gid int) (err error) //sys Fstatfs(fd int, buf *Statfs_t) (err error) -//sys fstatat(dirfd int, path string, stat *Stat_t, flags int) (err error) = SYS_NEWFSTATAT //sys Ftruncate(fd int, length int64) (err error) //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Lchown(path string, uid int, gid int) (err error) @@ -38,7 +32,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Statfs(path string, buf *Statfs_t) (err error) @@ -94,12 +87,6 @@ func Time(t *Time_t) (tt Time_t, err error) { //sys Utime(path string, buf *Utimbuf) (err error) //sys utimes(path string, times *[2]Timeval) (err error) -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func setTimespec(sec, nsec int64) Timespec { return Timespec{Sec: sec, Nsec: nsec} } @@ -138,10 +125,22 @@ type stat_t struct { Blocks int64 } +//sys fstatatInternal(dirfd int, path string, stat *stat_t, flags int) (err error) = SYS_NEWFSTATAT //sys fstat(fd int, st *stat_t) (err error) //sys lstat(path string, st *stat_t) (err error) //sys stat(path string, st *stat_t) (err error) +func fstatat(fd int, path string, s *Stat_t, flags int) (err error) { + st := &stat_t{} + err = fstatatInternal(fd, path, st, flags) + fillStat_t(s, st) + return +} + +func Fstatat(fd int, path string, s *Stat_t, flags int) (err error) { + return fstatat(fd, path, s, flags) +} + func Fstat(fd int, s *Stat_t) (err error) { st := &stat_t{} err = fstat(fd, st) diff --git a/src/syscall/syscall_linux_mipsx.go b/src/syscall/syscall_linux_mipsx.go index 7d4f8f22..aa087926 100644 --- a/src/syscall/syscall_linux_mipsx.go +++ b/src/syscall/syscall_linux_mipsx.go @@ -117,96 +117,6 @@ func mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int6 return mmap2(addr, length, prot, flags, fd, page) } -const rlimInf32 = ^uint32(0) -const rlimInf64 = ^uint64(0) - -type rlimit32 struct { - Cur uint32 - Max uint32 -} - -//sysnb getrlimit(resource int, rlim *rlimit32) (err error) = SYS_GETRLIMIT - -func Getrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - err = getrlimit(resource, &rl) - if err != nil { - return - } - - if rl.Cur == rlimInf32 { - rlim.Cur = rlimInf64 - } else { - rlim.Cur = uint64(rl.Cur) - } - - if rl.Max == rlimInf32 { - rlim.Max = rlimInf64 - } else { - rlim.Max = uint64(rl.Max) - } - return -} - -//sysnb setrlimit1(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func setrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit1(resource, &rl) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return uint64(r.Regs[64]) } func (r *PtraceRegs) SetPC(pc uint64) { r.Regs[64] = uint32(pc) } diff --git a/src/syscall/syscall_linux_ppc64x.go b/src/syscall/syscall_linux_ppc64x.go index 13c184c4..12424023 100644 --- a/src/syscall/syscall_linux_ppc64x.go +++ b/src/syscall/syscall_linux_ppc64x.go @@ -6,10 +6,6 @@ package syscall -import ( - "unsafe" -) - const ( _SYS_setgroups = SYS_SETGROUPS _SYS_clone3 = 435 @@ -27,7 +23,6 @@ const ( //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) = SYS_UGETRLIMIT //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Ioperm(from int, num int, on int) (err error) @@ -44,7 +39,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Stat(path string, stat *Stat_t) (err error) @@ -73,12 +67,6 @@ const ( //sys Utime(path string, buf *Utimbuf) (err error) //sys utimes(path string, times *[2]Timeval) (err error) -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func setTimespec(sec, nsec int64) Timespec { return Timespec{Sec: sec, Nsec: nsec} } diff --git a/src/syscall/syscall_linux_riscv64.go b/src/syscall/syscall_linux_riscv64.go index 00872a74..c9159762 100644 --- a/src/syscall/syscall_linux_riscv64.go +++ b/src/syscall/syscall_linux_riscv64.go @@ -27,7 +27,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sys Listen(s int, n int) (err error) //sys pread(fd int, p []byte, offset int64) (n int, err error) = SYS_PREAD64 @@ -37,7 +36,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) @@ -144,12 +142,6 @@ func utimes(path string, tv *[2]Timeval) (err error) { return utimensat(_AT_FDCWD, path, (*[2]Timespec)(unsafe.Pointer(&ts[0])), 0) } -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return r.Pc } func (r *PtraceRegs) SetPC(pc uint64) { r.Pc = pc } diff --git a/src/syscall/syscall_linux_s390x.go b/src/syscall/syscall_linux_s390x.go index ea667ec1..65c86140 100644 --- a/src/syscall/syscall_linux_s390x.go +++ b/src/syscall/syscall_linux_s390x.go @@ -23,7 +23,6 @@ const ( //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) = SYS_GETRLIMIT //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Lchown(path string, uid int, gid int) (err error) @@ -37,7 +36,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Stat(path string, stat *Stat_t) (err error) //sys Statfs(path string, buf *Statfs_t) (err error) @@ -244,12 +242,6 @@ func Shutdown(s, how int) (err error) { return } -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return r.Psw.Addr } func (r *PtraceRegs) SetPC(pc uint64) { r.Psw.Addr = pc } diff --git a/src/syscall/syscall_linux_test.go b/src/syscall/syscall_linux_test.go index d7543ceb..c719be52 100644 --- a/src/syscall/syscall_linux_test.go +++ b/src/syscall/syscall_linux_test.go @@ -5,6 +5,7 @@ package syscall_test import ( + "context" "fmt" "internal/testenv" "io" @@ -22,28 +23,6 @@ import ( "unsafe" ) -// chtmpdir changes the working directory to a new temporary directory and -// provides a cleanup function. Used when PWD is read-only. -func chtmpdir(t *testing.T) func() { - oldwd, err := os.Getwd() - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - d, err := os.MkdirTemp("", "test") - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - if err := os.Chdir(d); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - return func() { - if err := os.Chdir(oldwd); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - os.RemoveAll(d) - } -} - func touch(t *testing.T, name string) { f, err := os.Create(name) if err != nil { @@ -63,7 +42,7 @@ const ( ) func TestFaccessat(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) touch(t, "file1") err := syscall.Faccessat(_AT_FDCWD, "file1", _R_OK, 0) @@ -115,7 +94,7 @@ func TestFaccessat(t *testing.T) { } func TestFchmodat(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) touch(t, "file1") os.Symlink("file1", "symlink1") @@ -212,12 +191,8 @@ func TestSyscallNoError(t *testing.T) { } // Copy the test binary to a location that a non-root user can read/execute - // after we drop privileges - tempDir, err := os.MkdirTemp("", "TestSyscallNoError") - if err != nil { - t.Fatalf("cannot create temporary directory: %v", err) - } - defer os.RemoveAll(tempDir) + // after we drop privileges. + tempDir := t.TempDir() os.Chmod(tempDir, 0755) tmpBinary := filepath.Join(tempDir, filepath.Base(os.Args[0])) @@ -524,6 +499,9 @@ func TestSetuidEtc(t *testing.T) { if syscall.Getuid() != 0 { t.Skip("skipping root only test") } + if syscall.Getgid() != 0 { + t.Skip("skipping the test when root's gid is not default value 0") + } if testing.Short() && testenv.Builder() != "" && os.Getenv("USER") == "swarming" { // The Go build system's swarming user is known not to be root. // Unfortunately, it sometimes appears as root due the current @@ -720,3 +698,197 @@ func TestPrlimitOtherProcess(t *testing.T) { t.Fatalf("origRlimitNofile got=%v, want=%v", rlimLater, rlimOrig) } } + +const magicRlimitValue = 42 + +// TestPrlimitFileLimit tests that we can start a Go program, use +// prlimit to change its NOFILE limit, and have that updated limit be +// seen by children. See issue #66797. +func TestPrlimitFileLimit(t *testing.T) { + switch os.Getenv("GO_WANT_HELPER_PROCESS") { + case "prlimit1": + testPrlimitFileLimitHelper1(t) + return + case "prlimit2": + testPrlimitFileLimitHelper2(t) + return + } + + origRlimitNofile := syscall.GetInternalOrigRlimitNofile() + defer origRlimitNofile.Store(origRlimitNofile.Load()) + + // Set our rlimit to magic+1/max. + // That will also become the rlimit of the child. + + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + max := lim.Max + + lim = syscall.Rlimit{ + Cur: magicRlimitValue + 1, + Max: max, + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + + r1, w1, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r1.Close() + defer w1.Close() + + r2, w2, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r2.Close() + defer w2.Close() + + var output strings.Builder + + const arg = "-test.run=^TestPrlimitFileLimit$" + cmd := testenv.CommandContext(t, ctx, exe, arg, "-test.v") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=prlimit1") + cmd.ExtraFiles = []*os.File{r1, w2} + cmd.Stdout = &output + cmd.Stderr = &output + + t.Logf("running %s %s", exe, arg) + + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + // Wait for the child to start. + b := make([]byte, 1) + if n, err := r2.Read(b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("read %d bytes, want 1", n) + } + + // Set the child's prlimit. + lim = syscall.Rlimit{ + Cur: magicRlimitValue, + Max: max, + } + if err := syscall.Prlimit(cmd.Process.Pid, syscall.RLIMIT_NOFILE, &lim, nil); err != nil { + t.Fatalf("Prlimit failed: %v", err) + } + + // Tell the child to continue. + if n, err := w1.Write(b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("wrote %d bytes, want 1", n) + } + + err = cmd.Wait() + if output.Len() > 0 { + t.Logf("%s", output.String()) + } + + if err != nil { + t.Errorf("child failed: %v", err) + } +} + +// testPrlimitFileLimitHelper1 is run by TestPrlimitFileLimit. +func testPrlimitFileLimitHelper1(t *testing.T) { + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + t.Logf("helper1 rlimit is %v", lim) + t.Logf("helper1 cached rlimit is %v", syscall.OrigRlimitNofile()) + + // Tell the parent that we are ready. + b := []byte{0} + if n, err := syscall.Write(4, b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("wrote %d bytes, want 1", n) + } + + // Wait for the parent to tell us that prlimit was used. + if n, err := syscall.Read(3, b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("read %d bytes, want 1", n) + } + + if err := syscall.Close(3); err != nil { + t.Errorf("Close(3): %v", err) + } + if err := syscall.Close(4); err != nil { + t.Errorf("Close(4): %v", err) + } + + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + t.Logf("after prlimit helper1 rlimit is %v", lim) + t.Logf("after prlimit helper1 cached rlimit is %v", syscall.OrigRlimitNofile()) + + // Start the grandchild, which should see the rlimit + // set by the prlimit called by the parent. + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + + const arg = "-test.run=^TestPrlimitFileLimit$" + cmd := testenv.CommandContext(t, ctx, exe, arg, "-test.v") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=prlimit2") + t.Logf("running %s %s", exe, arg) + out, err := cmd.CombinedOutput() + if len(out) > 0 { + t.Logf("%s", out) + } + if err != nil { + t.Errorf("grandchild failed: %v", err) + } else { + fmt.Println("OK") + } +} + +// testPrlimitFileLimitHelper2 is run by testPrlimitFileLimit1. +func testPrlimitFileLimitHelper2(t *testing.T) { + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + + t.Logf("helper2 rlimit is %v", lim) + cached := syscall.OrigRlimitNofile() + t.Logf("helper2 cached rlimit is %v", cached) + + // The value return by Getrlimit will have been adjusted. + // We should have cached the value set by prlimit called by the parent. + + if cached == nil { + t.Fatal("no cached rlimit") + } else if cached.Cur != magicRlimitValue { + t.Fatalf("cached rlimit is %d, want %d", cached.Cur, magicRlimitValue) + } + + fmt.Println("OK") +} diff --git a/src/syscall/syscall_openbsd_libc.go b/src/syscall/syscall_openbsd_libc.go index 5dea268c..13311398 100644 --- a/src/syscall/syscall_openbsd_libc.go +++ b/src/syscall/syscall_openbsd_libc.go @@ -79,7 +79,6 @@ func syscall9X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, //sysnb execve(path *byte, argv **byte, envp **byte) (err error) //sysnb exit(res int) (err error) //sys ptrace(request int, pid int, addr uintptr, data uintptr) (err error) -//sysnb getentropy(p []byte) (err error) //sys fstatat(fd int, path string, stat *Stat_t, flags int) (err error) //sys unlinkat(fd int, path string, flags int) (err error) //sys openat(fd int, path string, flags int, perm uint32) (fdret int, err error) diff --git a/src/syscall/syscall_wasip1.go b/src/syscall/syscall_wasip1.go index 84c6bddc..c9225293 100644 --- a/src/syscall/syscall_wasip1.go +++ b/src/syscall/syscall_wasip1.go @@ -216,12 +216,14 @@ const ( O_WRONLY = 1 O_RDWR = 2 - O_CREAT = 0100 - O_CREATE = O_CREAT - O_TRUNC = 01000 - O_APPEND = 02000 - O_EXCL = 0200 - O_SYNC = 010000 + O_CREAT = 0100 + O_CREATE = O_CREAT + O_TRUNC = 01000 + O_APPEND = 02000 + O_EXCL = 0200 + O_SYNC = 010000 + O_DIRECTORY = 020000 + O_NOFOLLOW = 0400 O_CLOEXEC = 0 ) @@ -307,7 +309,7 @@ func (w WaitStatus) TrapCause() int { return 0 } // Rusage is a placeholder to allow compilation of the [os/exec] package // because we need Go programs to be portable across platforms. WASI does -// not have a mechanism to to spawn processes so there is no reason for an +// not have a mechanism to spawn processes so there is no reason for an // application to take a dependency on this type. type Rusage struct { Utime Timeval @@ -316,7 +318,7 @@ type Rusage struct { // ProcAttr is a placeholder to allow compilation of the [os/exec] package // because we need Go programs to be portable across platforms. WASI does -// not have a mechanism to to spawn processes so there is no reason for an +// not have a mechanism to spawn processes so there is no reason for an // application to take a dependency on this type. type ProcAttr struct { Dir string @@ -381,7 +383,7 @@ func Getppid() int { func Gettimeofday(tv *Timeval) error { var time timestamp - if errno := clock_time_get(clockRealtime, 1e3, unsafe.Pointer(&time)); errno != 0 { + if errno := clock_time_get(clockRealtime, 1e3, &time); errno != 0 { return errno } tv.setTimestamp(time) @@ -463,7 +465,7 @@ const ( //go:wasmimport wasi_snapshot_preview1 clock_time_get //go:noescape -func clock_time_get(id clockid, precision timestamp, time unsafe.Pointer) Errno +func clock_time_get(id clockid, precision timestamp, time *timestamp) Errno func SetNonblock(fd int, nonblocking bool) error { flags, err := fd_fdstat_get_flags(fd) diff --git a/src/syscall/syscall_windows.go b/src/syscall/syscall_windows.go index d49ee522..05c29c7b 100644 --- a/src/syscall/syscall_windows.go +++ b/src/syscall/syscall_windows.go @@ -323,6 +323,7 @@ func NewCallbackCDecl(fn any) uintptr { //sys Process32First(snapshot Handle, procEntry *ProcessEntry32) (err error) = kernel32.Process32FirstW //sys Process32Next(snapshot Handle, procEntry *ProcessEntry32) (err error) = kernel32.Process32NextW //sys DeviceIoControl(handle Handle, ioControlCode uint32, inBuffer *byte, inBufferSize uint32, outBuffer *byte, outBufferSize uint32, bytesReturned *uint32, overlapped *Overlapped) (err error) +//sys setFileInformationByHandle(handle Handle, fileInformationClass uint32, buf unsafe.Pointer, bufsize uint32) (err error) = kernel32.SetFileInformationByHandle // This function returns 1 byte BOOLEAN rather than the 4 byte BOOL. //sys CreateSymbolicLink(symlinkfilename *uint16, targetfilename *uint16, flags uint32) (err error) [failretval&0xff==0] = CreateSymbolicLinkW //sys CreateHardLink(filename *uint16, existingfilename *uint16, reserved uintptr) (err error) [failretval&0xff==0] = CreateHardLinkW @@ -340,16 +341,16 @@ func makeInheritSa() *SecurityAttributes { return &sa } -func Open(path string, mode int, perm uint32) (fd Handle, err error) { - if len(path) == 0 { +func Open(name string, flag int, perm uint32) (fd Handle, err error) { + if len(name) == 0 { return InvalidHandle, ERROR_FILE_NOT_FOUND } - pathp, err := UTF16PtrFromString(path) + namep, err := UTF16PtrFromString(name) if err != nil { return InvalidHandle, err } var access uint32 - switch mode & (O_RDONLY | O_WRONLY | O_RDWR) { + switch flag & (O_RDONLY | O_WRONLY | O_RDWR) { case O_RDONLY: access = GENERIC_READ case O_WRONLY: @@ -357,64 +358,71 @@ func Open(path string, mode int, perm uint32) (fd Handle, err error) { case O_RDWR: access = GENERIC_READ | GENERIC_WRITE } - if mode&O_CREAT != 0 { + if flag&O_CREAT != 0 { access |= GENERIC_WRITE } - if mode&O_APPEND != 0 { - access &^= GENERIC_WRITE - access |= FILE_APPEND_DATA + if flag&O_APPEND != 0 { + // Remove GENERIC_WRITE unless O_TRUNC is set, in which case we need it to truncate the file. + // We can't just remove FILE_WRITE_DATA because GENERIC_WRITE without FILE_WRITE_DATA + // starts appending at the beginning of the file rather than at the end. + if flag&O_TRUNC == 0 { + access &^= GENERIC_WRITE + } + // Set all access rights granted by GENERIC_WRITE except for FILE_WRITE_DATA. + access |= FILE_APPEND_DATA | FILE_WRITE_ATTRIBUTES | _FILE_WRITE_EA | STANDARD_RIGHTS_WRITE | SYNCHRONIZE } sharemode := uint32(FILE_SHARE_READ | FILE_SHARE_WRITE) var sa *SecurityAttributes - if mode&O_CLOEXEC == 0 { + if flag&O_CLOEXEC == 0 { sa = makeInheritSa() } + // We don't use CREATE_ALWAYS, because when opening a file with + // FILE_ATTRIBUTE_READONLY these will replace an existing file + // with a new, read-only one. See https://go.dev/issue/38225. + // + // Instead, we ftruncate the file after opening when O_TRUNC is set. var createmode uint32 switch { - case mode&(O_CREAT|O_EXCL) == (O_CREAT | O_EXCL): + case flag&(O_CREAT|O_EXCL) == (O_CREAT | O_EXCL): createmode = CREATE_NEW - case mode&(O_CREAT|O_TRUNC) == (O_CREAT | O_TRUNC): - createmode = CREATE_ALWAYS - case mode&O_CREAT == O_CREAT: + case flag&O_CREAT == O_CREAT: createmode = OPEN_ALWAYS - case mode&O_TRUNC == O_TRUNC: - createmode = TRUNCATE_EXISTING default: createmode = OPEN_EXISTING } var attrs uint32 = FILE_ATTRIBUTE_NORMAL if perm&S_IWRITE == 0 { attrs = FILE_ATTRIBUTE_READONLY - if createmode == CREATE_ALWAYS { - // We have been asked to create a read-only file. - // If the file already exists, the semantics of - // the Unix open system call is to preserve the - // existing permissions. If we pass CREATE_ALWAYS - // and FILE_ATTRIBUTE_READONLY to CreateFile, - // and the file already exists, CreateFile will - // change the file permissions. - // Avoid that to preserve the Unix semantics. - h, e := CreateFile(pathp, access, sharemode, sa, TRUNCATE_EXISTING, FILE_ATTRIBUTE_NORMAL, 0) - switch e { - case ERROR_FILE_NOT_FOUND, _ERROR_BAD_NETPATH, ERROR_PATH_NOT_FOUND: - // File does not exist. These are the same - // errors as Errno.Is checks for ErrNotExist. - // Carry on to create the file. - default: - // Success or some different error. - return h, e - } - } } - if createmode == OPEN_EXISTING && access == GENERIC_READ { - // Necessary for opening directory handles. + if flag&O_WRONLY == 0 && flag&O_RDWR == 0 { + // We might be opening or creating a directory. + // CreateFile requires FILE_FLAG_BACKUP_SEMANTICS + // to work with directories. attrs |= FILE_FLAG_BACKUP_SEMANTICS } - if mode&O_SYNC != 0 { + if flag&O_SYNC != 0 { const _FILE_FLAG_WRITE_THROUGH = 0x80000000 attrs |= _FILE_FLAG_WRITE_THROUGH } - return CreateFile(pathp, access, sharemode, sa, createmode, attrs, 0) + h, err := CreateFile(namep, access, sharemode, sa, createmode, attrs, 0) + if err != nil { + if err == ERROR_ACCESS_DENIED && (flag&O_WRONLY != 0 || flag&O_RDWR != 0) { + // We should return EISDIR when we are trying to open a directory with write access. + fa, e1 := GetFileAttributes(namep) + if e1 == nil && fa&FILE_ATTRIBUTE_DIRECTORY != 0 { + err = EISDIR + } + } + return InvalidHandle, err + } + if flag&O_TRUNC == O_TRUNC { + err = Ftruncate(h, 0) + if err != nil { + CloseHandle(h) + return InvalidHandle, err + } + } + return h, nil } func Read(fd Handle, p []byte) (n int, err error) { @@ -610,20 +618,13 @@ func ComputerName() (name string, err error) { } func Ftruncate(fd Handle, length int64) (err error) { - curoffset, e := Seek(fd, 0, 1) - if e != nil { - return e + type _FILE_END_OF_FILE_INFO struct { + EndOfFile int64 } - defer Seek(fd, curoffset, 0) - _, e = Seek(fd, length, 0) - if e != nil { - return e - } - e = SetEndOfFile(fd) - if e != nil { - return e - } - return nil + const FileEndOfFileInfo = 6 + var info _FILE_END_OF_FILE_INFO + info.EndOfFile = length + return setFileInformationByHandle(fd, FileEndOfFileInfo, unsafe.Pointer(&info), uint32(unsafe.Sizeof(info))) } func Gettimeofday(tv *Timeval) (err error) { diff --git a/src/syscall/syscall_windows_test.go b/src/syscall/syscall_windows_test.go index a6c6eff3..882a2796 100644 --- a/src/syscall/syscall_windows_test.go +++ b/src/syscall/syscall_windows_test.go @@ -15,25 +15,40 @@ import ( "testing" ) -func TestOpen_Dir(t *testing.T) { - dir := t.TempDir() +func TestOpen(t *testing.T) { + t.Parallel() - h, err := syscall.Open(dir, syscall.O_RDONLY, 0) + dir := t.TempDir() + file := filepath.Join(dir, "a") + f, err := os.Create(file) if err != nil { - t.Fatalf("Open failed: %v", err) + t.Fatal(err) } - syscall.CloseHandle(h) - h, err = syscall.Open(dir, syscall.O_RDONLY|syscall.O_TRUNC, 0) - if err == nil { - t.Error("Open should have failed") - } else { - syscall.CloseHandle(h) + f.Close() + + tests := []struct { + path string + flag int + err error + }{ + {dir, syscall.O_RDONLY, nil}, + {dir, syscall.O_CREAT, nil}, + {dir, syscall.O_RDONLY | syscall.O_CREAT, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE | os.O_TRUNC, nil}, + {dir, syscall.O_RDONLY | syscall.O_TRUNC, syscall.ERROR_ACCESS_DENIED}, + {dir, syscall.O_WRONLY | syscall.O_RDWR, syscall.EISDIR}, + {dir, syscall.O_WRONLY, syscall.EISDIR}, + {dir, syscall.O_RDWR, syscall.EISDIR}, } - h, err = syscall.Open(dir, syscall.O_RDONLY|syscall.O_CREAT, 0) - if err == nil { - t.Error("Open should have failed") - } else { - syscall.CloseHandle(h) + for i, tt := range tests { + h, err := syscall.Open(tt.path, tt.flag, 0o660) + if err == nil { + syscall.CloseHandle(h) + } + if err != tt.err { + t.Errorf("%d: Open got %q, want %q", i, err, tt.err) + } } } @@ -182,12 +197,14 @@ int main(int argc, char *argv[]) func TestGetwd_DoesNotPanicWhenPathIsLong(t *testing.T) { // Regression test for https://github.com/golang/go/issues/60051. + tmp := t.TempDir() + t.Chdir(tmp) // The length of a filename is also limited, so we can't reproduce the // crash by creating a single directory with a very long name; we need two // layers. a200 := strings.Repeat("a", 200) - dirname := filepath.Join(t.TempDir(), a200, a200) + dirname := filepath.Join(tmp, a200, a200) err := os.MkdirAll(dirname, 0o700) if err != nil { @@ -197,9 +214,6 @@ func TestGetwd_DoesNotPanicWhenPathIsLong(t *testing.T) { if err != nil { t.Skipf("Chdir failed: %v", err) } - // Change out of the temporary directory so that we don't inhibit its - // removal during test cleanup. - defer os.Chdir(`\`) syscall.Getwd() } diff --git a/src/syscall/tables_wasip1.go b/src/syscall/tables_wasip1.go index 973a56e2..be178c1c 100644 --- a/src/syscall/tables_wasip1.go +++ b/src/syscall/tables_wasip1.go @@ -86,6 +86,7 @@ const ( ETXTBSY Errno = 74 EXDEV Errno = 75 ENOTCAPABLE Errno = 76 + EBADFD Errno = 77 // needed by src/net/error_unix_test.go EOPNOTSUPP = ENOTSUP ) @@ -100,6 +101,7 @@ var errorstr = [...]string{ EAGAIN: "Try again", EALREADY: "Socket already connected", EBADF: "Bad file number", + EBADFD: "file descriptor in bad state", EBADMSG: "Trying to read unreadable message", EBUSY: "Device or resource busy", ECANCELED: "Operation canceled.", diff --git a/src/syscall/types_windows.go b/src/syscall/types_windows.go index 37d0eff0..f08ebe01 100644 --- a/src/syscall/types_windows.go +++ b/src/syscall/types_windows.go @@ -34,18 +34,21 @@ const ( const ( // Invented values to support what package os expects. - O_RDONLY = 0x00000 - O_WRONLY = 0x00001 - O_RDWR = 0x00002 - O_CREAT = 0x00040 - O_EXCL = 0x00080 - O_NOCTTY = 0x00100 - O_TRUNC = 0x00200 - O_NONBLOCK = 0x00800 - O_APPEND = 0x00400 - O_SYNC = 0x01000 - O_ASYNC = 0x02000 - O_CLOEXEC = 0x80000 + O_RDONLY = 0x00000 + O_WRONLY = 0x00001 + O_RDWR = 0x00002 + O_CREAT = 0x00040 + O_EXCL = 0x00080 + O_NOCTTY = 0x00100 + O_TRUNC = 0x00200 + O_NONBLOCK = 0x00800 + O_APPEND = 0x00400 + O_SYNC = 0x01000 + O_ASYNC = 0x02000 + O_CLOEXEC = 0x80000 + o_DIRECTORY = 0x100000 // used by internal/syscall/windows + o_NOFOLLOW_ANY = 0x20000000 // used by internal/syscall/windows + o_OPEN_REPARSE = 0x40000000 // used by internal/syscall/windows ) const ( @@ -91,6 +94,7 @@ const ( FILE_LIST_DIRECTORY = 0x00000001 FILE_APPEND_DATA = 0x00000004 + _FILE_WRITE_EA = 0x00000010 FILE_WRITE_ATTRIBUTES = 0x00000100 FILE_SHARE_READ = 0x00000001 diff --git a/src/syscall/zerrors_darwin_amd64.go b/src/syscall/zerrors_darwin_amd64.go index ecbe89c5..bad1f889 100644 --- a/src/syscall/zerrors_darwin_amd64.go +++ b/src/syscall/zerrors_darwin_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && darwin - package syscall const ( diff --git a/src/syscall/zerrors_darwin_arm64.go b/src/syscall/zerrors_darwin_arm64.go index fa7cb845..53236ca4 100644 --- a/src/syscall/zerrors_darwin_arm64.go +++ b/src/syscall/zerrors_darwin_arm64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build arm64 && darwin - package syscall const ( diff --git a/src/syscall/zerrors_dragonfly_amd64.go b/src/syscall/zerrors_dragonfly_amd64.go index bca2f50c..d87ab132 100644 --- a/src/syscall/zerrors_dragonfly_amd64.go +++ b/src/syscall/zerrors_dragonfly_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && dragonfly - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_386.go b/src/syscall/zerrors_freebsd_386.go index b1441e76..c59b7dae 100644 --- a/src/syscall/zerrors_freebsd_386.go +++ b/src/syscall/zerrors_freebsd_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && freebsd - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_amd64.go b/src/syscall/zerrors_freebsd_amd64.go index 3aed0049..305b8c66 100644 --- a/src/syscall/zerrors_freebsd_amd64.go +++ b/src/syscall/zerrors_freebsd_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && freebsd - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_arm.go b/src/syscall/zerrors_freebsd_arm.go index e1f91ff8..9e082e30 100644 --- a/src/syscall/zerrors_freebsd_arm.go +++ b/src/syscall/zerrors_freebsd_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm && freebsd - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_arm64.go b/src/syscall/zerrors_freebsd_arm64.go index d0cb6c8a..305b8c66 100644 --- a/src/syscall/zerrors_freebsd_arm64.go +++ b/src/syscall/zerrors_freebsd_arm64.go @@ -1,8 +1,6 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -//go:build freebsd && arm64 - // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go diff --git a/src/syscall/zerrors_freebsd_riscv64.go b/src/syscall/zerrors_freebsd_riscv64.go index 7aa9aa98..305b8c66 100644 --- a/src/syscall/zerrors_freebsd_riscv64.go +++ b/src/syscall/zerrors_freebsd_riscv64.go @@ -1,8 +1,6 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -//go:build freebsd && riscv64 - // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go diff --git a/src/syscall/zerrors_linux_386.go b/src/syscall/zerrors_linux_386.go index 045a4166..9d4ecdeb 100644 --- a/src/syscall/zerrors_linux_386.go +++ b/src/syscall/zerrors_linux_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_amd64.go b/src/syscall/zerrors_linux_amd64.go index 4eb44746..a8b67801 100644 --- a/src/syscall/zerrors_linux_amd64.go +++ b/src/syscall/zerrors_linux_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_arm.go b/src/syscall/zerrors_linux_arm.go index a5f925ec..285d26a5 100644 --- a/src/syscall/zerrors_linux_arm.go +++ b/src/syscall/zerrors_linux_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_arm64.go b/src/syscall/zerrors_linux_arm64.go index ec8ac070..d32a290a 100644 --- a/src/syscall/zerrors_linux_arm64.go +++ b/src/syscall/zerrors_linux_arm64.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm64 && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_mips.go b/src/syscall/zerrors_linux_mips.go index 3fe5c007..81657d1c 100644 --- a/src/syscall/zerrors_linux_mips.go +++ b/src/syscall/zerrors_linux_mips.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go package syscall diff --git a/src/syscall/zerrors_linux_mips64.go b/src/syscall/zerrors_linux_mips64.go index 74a1843e..6953c924 100644 --- a/src/syscall/zerrors_linux_mips64.go +++ b/src/syscall/zerrors_linux_mips64.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs _const.go package syscall diff --git a/src/syscall/zerrors_linux_mips64le.go b/src/syscall/zerrors_linux_mips64le.go index 74a1843e..6953c924 100644 --- a/src/syscall/zerrors_linux_mips64le.go +++ b/src/syscall/zerrors_linux_mips64le.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs _const.go package syscall diff --git a/src/syscall/zerrors_linux_mipsle.go b/src/syscall/zerrors_linux_mipsle.go index 3fe5c007..81657d1c 100644 --- a/src/syscall/zerrors_linux_mipsle.go +++ b/src/syscall/zerrors_linux_mipsle.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go package syscall diff --git a/src/syscall/zerrors_linux_ppc64.go b/src/syscall/zerrors_linux_ppc64.go index b63daea8..f0661a24 100644 --- a/src/syscall/zerrors_linux_ppc64.go +++ b/src/syscall/zerrors_linux_ppc64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build ppc64 && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_ppc64le.go b/src/syscall/zerrors_linux_ppc64le.go index 01f8adb0..73849d49 100644 --- a/src/syscall/zerrors_linux_ppc64le.go +++ b/src/syscall/zerrors_linux_ppc64le.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build ppc64le && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_riscv64.go b/src/syscall/zerrors_linux_riscv64.go index f4b1d9ae..760d8742 100644 --- a/src/syscall/zerrors_linux_riscv64.go +++ b/src/syscall/zerrors_linux_riscv64.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go package syscall diff --git a/src/syscall/zerrors_linux_s390x.go b/src/syscall/zerrors_linux_s390x.go index 8b99a609..9abad9ed 100644 --- a/src/syscall/zerrors_linux_s390x.go +++ b/src/syscall/zerrors_linux_s390x.go @@ -1,7 +1,7 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go package syscall diff --git a/src/syscall/zerrors_netbsd_386.go b/src/syscall/zerrors_netbsd_386.go index b2fcb657..6f21b562 100644 --- a/src/syscall/zerrors_netbsd_386.go +++ b/src/syscall/zerrors_netbsd_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_netbsd_amd64.go b/src/syscall/zerrors_netbsd_amd64.go index dc52c3ca..a404ac21 100644 --- a/src/syscall/zerrors_netbsd_amd64.go +++ b/src/syscall/zerrors_netbsd_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_netbsd_arm.go b/src/syscall/zerrors_netbsd_arm.go index 3137e18a..fdf0ca0c 100644 --- a/src/syscall/zerrors_netbsd_arm.go +++ b/src/syscall/zerrors_netbsd_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh -marm // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -marm _const.go -//go:build arm && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_netbsd_arm64.go b/src/syscall/zerrors_netbsd_arm64.go index cc1b0088..a404ac21 100644 --- a/src/syscall/zerrors_netbsd_arm64.go +++ b/src/syscall/zerrors_netbsd_arm64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build arm64 && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_openbsd_386.go b/src/syscall/zerrors_openbsd_386.go index a37d6d16..4bdae34b 100644 --- a/src/syscall/zerrors_openbsd_386.go +++ b/src/syscall/zerrors_openbsd_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && openbsd - package syscall const ( diff --git a/src/syscall/zerrors_openbsd_amd64.go b/src/syscall/zerrors_openbsd_amd64.go index 812fd950..69c68502 100644 --- a/src/syscall/zerrors_openbsd_amd64.go +++ b/src/syscall/zerrors_openbsd_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && openbsd - package syscall const ( diff --git a/src/syscall/zerrors_openbsd_arm.go b/src/syscall/zerrors_openbsd_arm.go index 2e19672b..dc6379b3 100644 --- a/src/syscall/zerrors_openbsd_arm.go +++ b/src/syscall/zerrors_openbsd_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm && openbsd - package syscall const ( diff --git a/src/syscall/zerrors_solaris_amd64.go b/src/syscall/zerrors_solaris_amd64.go index b2c81d9a..094a78dc 100644 --- a/src/syscall/zerrors_solaris_amd64.go +++ b/src/syscall/zerrors_solaris_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && solaris - package syscall const ( diff --git a/src/syscall/zsyscall_linux_386.go b/src/syscall/zsyscall_linux_386.go index 661cfe7b..762fb007 100644 --- a/src/syscall/zsyscall_linux_386.go +++ b/src/syscall/zsyscall_linux_386.go @@ -1395,26 +1395,6 @@ func EpollWait(epfd int, events []EpollEvent, msec int) (n int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func futimesat(dirfd int, path string, times *[2]Timeval) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_linux_amd64.go b/src/syscall/zsyscall_linux_amd64.go index 9d057818..1249ff66 100644 --- a/src/syscall/zsyscall_linux_amd64.go +++ b/src/syscall/zsyscall_linux_amd64.go @@ -1160,16 +1160,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1336,16 +1326,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_arm.go b/src/syscall/zsyscall_linux_arm.go index a386120f..c359fe53 100644 --- a/src/syscall/zsyscall_linux_arm.go +++ b/src/syscall/zsyscall_linux_arm.go @@ -1,4 +1,4 @@ -// mksyscall.pl -l32 -arm -tags linux,arm syscall_linux.go syscall_linux_arm.go syscall_linux_accept.go +// mksyscall.pl -l32 -arm -tags linux,arm syscall_linux.go syscall_linux_arm.go // Code generated by the command above; DO NOT EDIT. //go:build linux && arm @@ -1595,34 +1595,3 @@ func EpollWait(epfd int, events []EpollEvent, msec int) (n int, err error) { } return } - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func accept(s int, rsa *RawSockaddrAny, addrlen *_Socklen) (fd int, err error) { - r0, _, e1 := Syscall(SYS_ACCEPT, uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) - fd = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/src/syscall/zsyscall_linux_arm64.go b/src/syscall/zsyscall_linux_arm64.go index a1c91071..6625cddb 100644 --- a/src/syscall/zsyscall_linux_arm64.go +++ b/src/syscall/zsyscall_linux_arm64.go @@ -1182,16 +1182,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1306,16 +1296,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit1(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_mips.go b/src/syscall/zsyscall_linux_mips.go index 7e216b04..0bd4a806 100644 --- a/src/syscall/zsyscall_linux_mips.go +++ b/src/syscall/zsyscall_linux_mips.go @@ -1670,23 +1670,3 @@ func mmap2(addr uintptr, length uintptr, prot int, flags int, fd int, pageOffset } return } - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/src/syscall/zsyscall_linux_mips64.go b/src/syscall/zsyscall_linux_mips64.go index 8c894b10..449088c8 100644 --- a/src/syscall/zsyscall_linux_mips64.go +++ b/src/syscall/zsyscall_linux_mips64.go @@ -1116,21 +1116,6 @@ func Fstatfs(fd int, buf *Statfs_t) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func fstatat(dirfd int, path string, stat *Stat_t, flags int) (err error) { - var _p0 *byte - _p0, err = BytePtrFromString(path) - if err != nil { - return - } - _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Ftruncate(fd int, length int64) (err error) { _, _, e1 := Syscall(SYS_FTRUNCATE, uintptr(fd), uintptr(length), 0) if e1 != 0 { @@ -1165,16 +1150,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1325,16 +1300,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { @@ -1658,6 +1623,21 @@ func utimes(path string, times *[2]Timeval) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func fstatatInternal(dirfd int, path string, stat *stat_t, flags int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func fstat(fd int, st *stat_t) (err error) { _, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(unsafe.Pointer(st)), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_mips64le.go b/src/syscall/zsyscall_linux_mips64le.go index 812a6ba8..048298a3 100644 --- a/src/syscall/zsyscall_linux_mips64le.go +++ b/src/syscall/zsyscall_linux_mips64le.go @@ -1116,21 +1116,6 @@ func Fstatfs(fd int, buf *Statfs_t) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func fstatat(dirfd int, path string, stat *Stat_t, flags int) (err error) { - var _p0 *byte - _p0, err = BytePtrFromString(path) - if err != nil { - return - } - _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Ftruncate(fd int, length int64) (err error) { _, _, e1 := Syscall(SYS_FTRUNCATE, uintptr(fd), uintptr(length), 0) if e1 != 0 { @@ -1165,16 +1150,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1325,16 +1300,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { @@ -1658,6 +1623,21 @@ func utimes(path string, times *[2]Timeval) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func fstatatInternal(dirfd int, path string, stat *stat_t, flags int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func fstat(fd int, st *stat_t) (err error) { _, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(unsafe.Pointer(st)), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_mipsle.go b/src/syscall/zsyscall_linux_mipsle.go index d32a8afa..7f880e53 100644 --- a/src/syscall/zsyscall_linux_mipsle.go +++ b/src/syscall/zsyscall_linux_mipsle.go @@ -1670,23 +1670,3 @@ func mmap2(addr uintptr, length uintptr, prot int, flags int, fd int, pageOffset } return } - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/src/syscall/zsyscall_linux_ppc64.go b/src/syscall/zsyscall_linux_ppc64.go index c321267b..c4ca96f2 100644 --- a/src/syscall/zsyscall_linux_ppc64.go +++ b/src/syscall/zsyscall_linux_ppc64.go @@ -1192,16 +1192,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_UGETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1398,16 +1388,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_ppc64le.go b/src/syscall/zsyscall_linux_ppc64le.go index 40475d76..a08471b4 100644 --- a/src/syscall/zsyscall_linux_ppc64le.go +++ b/src/syscall/zsyscall_linux_ppc64le.go @@ -1192,16 +1192,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_UGETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1398,16 +1388,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_riscv64.go b/src/syscall/zsyscall_linux_riscv64.go index dc74acfc..4098b790 100644 --- a/src/syscall/zsyscall_linux_riscv64.go +++ b/src/syscall/zsyscall_linux_riscv64.go @@ -1182,16 +1182,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1306,16 +1296,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_s390x.go b/src/syscall/zsyscall_linux_s390x.go index cc189d9e..81979169 100644 --- a/src/syscall/zsyscall_linux_s390x.go +++ b/src/syscall/zsyscall_linux_s390x.go @@ -1192,16 +1192,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1368,16 +1358,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) { r0, _, e1 := Syscall6(SYS_SPLICE, uintptr(rfd), uintptr(unsafe.Pointer(roff)), uintptr(wfd), uintptr(unsafe.Pointer(woff)), uintptr(len), uintptr(flags)) n = int64(r0) diff --git a/src/syscall/zsyscall_openbsd_386.go b/src/syscall/zsyscall_openbsd_386.go index d2bd3ea0..c8cf7f23 100644 --- a/src/syscall/zsyscall_openbsd_386.go +++ b/src/syscall/zsyscall_openbsd_386.go @@ -1854,26 +1854,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_386.s b/src/syscall/zsyscall_openbsd_386.s index 9a820e9f..f86ac2c0 100644 --- a/src/syscall/zsyscall_openbsd_386.s +++ b/src/syscall/zsyscall_openbsd_386.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_amd64.go b/src/syscall/zsyscall_openbsd_amd64.go index 170a74b4..9188756a 100644 --- a/src/syscall/zsyscall_openbsd_amd64.go +++ b/src/syscall/zsyscall_openbsd_amd64.go @@ -1854,26 +1854,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_amd64.s b/src/syscall/zsyscall_openbsd_amd64.s index 9b70dc09..4ec62202 100644 --- a/src/syscall/zsyscall_openbsd_amd64.s +++ b/src/syscall/zsyscall_openbsd_amd64.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_arm.go b/src/syscall/zsyscall_openbsd_arm.go index e75bd0b4..ecdfa636 100644 --- a/src/syscall/zsyscall_openbsd_arm.go +++ b/src/syscall/zsyscall_openbsd_arm.go @@ -1854,26 +1854,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_arm.s b/src/syscall/zsyscall_openbsd_arm.s index 0333377b..75251d03 100644 --- a/src/syscall/zsyscall_openbsd_arm.s +++ b/src/syscall/zsyscall_openbsd_arm.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_arm64.go b/src/syscall/zsyscall_openbsd_arm64.go index bc027b44..d28d3c5e 100644 --- a/src/syscall/zsyscall_openbsd_arm64.go +++ b/src/syscall/zsyscall_openbsd_arm64.go @@ -1854,26 +1854,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_arm64.s b/src/syscall/zsyscall_openbsd_arm64.s index 654e6c69..deea88ec 100644 --- a/src/syscall/zsyscall_openbsd_arm64.s +++ b/src/syscall/zsyscall_openbsd_arm64.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_ppc64.go b/src/syscall/zsyscall_openbsd_ppc64.go index 6808092a..0e6828bc 100644 --- a/src/syscall/zsyscall_openbsd_ppc64.go +++ b/src/syscall/zsyscall_openbsd_ppc64.go @@ -1854,26 +1854,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_ppc64.s b/src/syscall/zsyscall_openbsd_ppc64.s index 86a5745c..cc4eb952 100644 --- a/src/syscall/zsyscall_openbsd_ppc64.s +++ b/src/syscall/zsyscall_openbsd_ppc64.s @@ -340,9 +340,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 CALL libc_ptrace(SB) RET -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - CALL libc_getentropy(SB) - RET TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 CALL libc_fstatat(SB) RET diff --git a/src/syscall/zsyscall_openbsd_riscv64.go b/src/syscall/zsyscall_openbsd_riscv64.go index 2979ff78..92014707 100644 --- a/src/syscall/zsyscall_openbsd_riscv64.go +++ b/src/syscall/zsyscall_openbsd_riscv64.go @@ -1854,26 +1854,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_riscv64.s b/src/syscall/zsyscall_openbsd_riscv64.s index c8728190..14b7b232 100644 --- a/src/syscall/zsyscall_openbsd_riscv64.s +++ b/src/syscall/zsyscall_openbsd_riscv64.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_windows.go b/src/syscall/zsyscall_windows.go index a47b0900..e58a384a 100644 --- a/src/syscall/zsyscall_windows.go +++ b/src/syscall/zsyscall_windows.go @@ -152,6 +152,7 @@ var ( procSetEnvironmentVariableW = modkernel32.NewProc("SetEnvironmentVariableW") procSetFileAttributesW = modkernel32.NewProc("SetFileAttributesW") procSetFileCompletionNotificationModes = modkernel32.NewProc("SetFileCompletionNotificationModes") + procSetFileInformationByHandle = modkernel32.NewProc("SetFileInformationByHandle") procSetFilePointer = modkernel32.NewProc("SetFilePointer") procSetFileTime = modkernel32.NewProc("SetFileTime") procSetHandleInformation = modkernel32.NewProc("SetHandleInformation") @@ -1083,6 +1084,14 @@ func SetFileCompletionNotificationModes(handle Handle, flags uint8) (err error) return } +func setFileInformationByHandle(handle Handle, fileInformationClass uint32, buf unsafe.Pointer, bufsize uint32) (err error) { + r1, _, e1 := Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(handle), uintptr(fileInformationClass), uintptr(buf), uintptr(bufsize), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func SetFilePointer(handle Handle, lowoffset int32, highoffsetptr *int32, whence uint32) (newlowoffset uint32, err error) { r0, _, e1 := Syscall6(procSetFilePointer.Addr(), 4, uintptr(handle), uintptr(lowoffset), uintptr(unsafe.Pointer(highoffsetptr)), uintptr(whence), 0, 0) newlowoffset = uint32(r0) diff --git a/src/syscall/zsysnum_darwin_amd64.go b/src/syscall/zsysnum_darwin_amd64.go index 08e003f2..dcff3ca8 100644 --- a/src/syscall/zsysnum_darwin_amd64.go +++ b/src/syscall/zsysnum_darwin_amd64.go @@ -1,8 +1,6 @@ // mksysnum_darwin.pl /usr/include/sys/syscall.h // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && darwin - package syscall const ( diff --git a/src/syscall/zsysnum_darwin_arm64.go b/src/syscall/zsysnum_darwin_arm64.go index 71309bb4..d023dc3e 100644 --- a/src/syscall/zsysnum_darwin_arm64.go +++ b/src/syscall/zsysnum_darwin_arm64.go @@ -1,8 +1,6 @@ // mksysnum_darwin.pl /usr/include/sys/syscall.h // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && darwin - package syscall const ( diff --git a/src/syscall/zsysnum_dragonfly_amd64.go b/src/syscall/zsysnum_dragonfly_amd64.go index 03d4b06a..b9fb1c08 100644 --- a/src/syscall/zsysnum_dragonfly_amd64.go +++ b/src/syscall/zsysnum_dragonfly_amd64.go @@ -1,8 +1,6 @@ // mksysnum_dragonfly.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && dragonfly - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_386.go b/src/syscall/zsysnum_freebsd_386.go index ada885e4..348bcd01 100644 --- a/src/syscall/zsysnum_freebsd_386.go +++ b/src/syscall/zsysnum_freebsd_386.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build 386 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_amd64.go b/src/syscall/zsysnum_freebsd_amd64.go index 85a50ad9..348bcd01 100644 --- a/src/syscall/zsysnum_freebsd_amd64.go +++ b/src/syscall/zsysnum_freebsd_amd64.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_arm.go b/src/syscall/zsysnum_freebsd_arm.go index 7f6e3fc3..348bcd01 100644 --- a/src/syscall/zsysnum_freebsd_arm.go +++ b/src/syscall/zsysnum_freebsd_arm.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_arm64.go b/src/syscall/zsysnum_freebsd_arm64.go index bde0f3bc..43870907 100644 --- a/src/syscall/zsysnum_freebsd_arm64.go +++ b/src/syscall/zsysnum_freebsd_arm64.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_riscv64.go b/src/syscall/zsysnum_freebsd_riscv64.go index 48e70296..43870907 100644 --- a/src/syscall/zsysnum_freebsd_riscv64.go +++ b/src/syscall/zsysnum_freebsd_riscv64.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build riscv64 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_linux_386.go b/src/syscall/zsysnum_linux_386.go index 4966d2a9..e5e474fb 100644 --- a/src/syscall/zsysnum_linux_386.go +++ b/src/syscall/zsysnum_linux_386.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm/unistd_32.h // Code generated by the command above; DO NOT EDIT. -//go:build 386 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_amd64.go b/src/syscall/zsysnum_linux_amd64.go index 576c7c36..2a78e3a0 100644 --- a/src/syscall/zsysnum_linux_amd64.go +++ b/src/syscall/zsysnum_linux_amd64.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm/unistd_64.h // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_arm.go b/src/syscall/zsysnum_linux_arm.go index b0da97c6..58e3a781 100644 --- a/src/syscall/zsysnum_linux_arm.go +++ b/src/syscall/zsysnum_linux_arm.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_arm64.go b/src/syscall/zsysnum_linux_arm64.go index 0136d944..0ccb57aa 100644 --- a/src/syscall/zsysnum_linux_arm64.go +++ b/src/syscall/zsysnum_linux_arm64.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm-generic/unistd.h // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_ppc64.go b/src/syscall/zsysnum_linux_ppc64.go index cc964c23..c2e058b2 100644 --- a/src/syscall/zsysnum_linux_ppc64.go +++ b/src/syscall/zsysnum_linux_ppc64.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm/unistd.h // Code generated by the command above; DO NOT EDIT. -//go:build ppc64 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_ppc64le.go b/src/syscall/zsysnum_linux_ppc64le.go index 57bfb779..0c8de627 100644 --- a/src/syscall/zsysnum_linux_ppc64le.go +++ b/src/syscall/zsysnum_linux_ppc64le.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/powerpc64le-linux-gnu/asm/unistd.h // Code generated by the command above; DO NOT EDIT. -//go:build ppc64le && linux - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_386.go b/src/syscall/zsysnum_netbsd_386.go index 5696c4be..c28d0b4e 100644 --- a/src/syscall/zsysnum_netbsd_386.go +++ b/src/syscall/zsysnum_netbsd_386.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build 386 && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_amd64.go b/src/syscall/zsysnum_netbsd_amd64.go index 9fb85cdd..c28d0b4e 100644 --- a/src/syscall/zsysnum_netbsd_amd64.go +++ b/src/syscall/zsysnum_netbsd_amd64.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_arm.go b/src/syscall/zsysnum_netbsd_arm.go index e0e89946..c28d0b4e 100644 --- a/src/syscall/zsysnum_netbsd_arm.go +++ b/src/syscall/zsysnum_netbsd_arm.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_arm64.go b/src/syscall/zsysnum_netbsd_arm64.go index 9653364d..c28d0b4e 100644 --- a/src/syscall/zsysnum_netbsd_arm64.go +++ b/src/syscall/zsysnum_netbsd_arm64.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_openbsd_386.go b/src/syscall/zsysnum_openbsd_386.go index 3b12639b..622efe19 100644 --- a/src/syscall/zsysnum_openbsd_386.go +++ b/src/syscall/zsysnum_openbsd_386.go @@ -1,8 +1,6 @@ // mksysnum_openbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build 386 && openbsd - package syscall const ( diff --git a/src/syscall/zsysnum_openbsd_amd64.go b/src/syscall/zsysnum_openbsd_amd64.go index bce309dc..22d2ce38 100644 --- a/src/syscall/zsysnum_openbsd_amd64.go +++ b/src/syscall/zsysnum_openbsd_amd64.go @@ -1,8 +1,6 @@ // mksysnum_openbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && openbsd - package syscall const ( diff --git a/src/syscall/zsysnum_openbsd_arm.go b/src/syscall/zsysnum_openbsd_arm.go index 05aed707..80051fdf 100644 --- a/src/syscall/zsysnum_openbsd_arm.go +++ b/src/syscall/zsysnum_openbsd_arm.go @@ -1,8 +1,6 @@ // mksysnum_openbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && openbsd - package syscall const ( diff --git a/src/syscall/zsysnum_solaris_amd64.go b/src/syscall/zsysnum_solaris_amd64.go index ea244e53..43b3d8b4 100644 --- a/src/syscall/zsysnum_solaris_amd64.go +++ b/src/syscall/zsysnum_solaris_amd64.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 && solaris - package syscall // TODO(aram): remove these before Go 1.3. diff --git a/src/syscall/ztypes_darwin_amd64.go b/src/syscall/ztypes_darwin_amd64.go index 551edc70..8b05961b 100644 --- a/src/syscall/ztypes_darwin_amd64.go +++ b/src/syscall/ztypes_darwin_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_darwin.go - -//go:build amd64 && darwin +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_darwin.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_darwin_arm64.go b/src/syscall/ztypes_darwin_arm64.go index 46f78a97..8b05961b 100644 --- a/src/syscall/ztypes_darwin_arm64.go +++ b/src/syscall/ztypes_darwin_arm64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_darwin.go - -//go:build arm64 && darwin +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_darwin.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_dragonfly_amd64.go b/src/syscall/ztypes_dragonfly_amd64.go index ec519b72..eae8221e 100644 --- a/src/syscall/ztypes_dragonfly_amd64.go +++ b/src/syscall/ztypes_dragonfly_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_dragonfly.go - -//go:build amd64 && dragonfly +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_dragonfly.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_freebsd_386.go b/src/syscall/ztypes_freebsd_386.go index 70ae808e..bbb222e0 100644 --- a/src/syscall/ztypes_freebsd_386.go +++ b/src/syscall/ztypes_freebsd_386.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build 386 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_amd64.go b/src/syscall/ztypes_freebsd_amd64.go index 050432a6..3f97d1e4 100644 --- a/src/syscall/ztypes_freebsd_amd64.go +++ b/src/syscall/ztypes_freebsd_amd64.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build amd64 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_arm.go b/src/syscall/ztypes_freebsd_arm.go index ae5ed56c..f6b1ae4e 100644 --- a/src/syscall/ztypes_freebsd_arm.go +++ b/src/syscall/ztypes_freebsd_arm.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -fsigned-char types_freebsd.go -//go:build arm && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_arm64.go b/src/syscall/ztypes_freebsd_arm64.go index b3ab991f..3f97d1e4 100644 --- a/src/syscall/ztypes_freebsd_arm64.go +++ b/src/syscall/ztypes_freebsd_arm64.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build arm64 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_riscv64.go b/src/syscall/ztypes_freebsd_riscv64.go index 28895184..3f97d1e4 100644 --- a/src/syscall/ztypes_freebsd_riscv64.go +++ b/src/syscall/ztypes_freebsd_riscv64.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build riscv64 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_linux_386.go b/src/syscall/ztypes_linux_386.go index 79a46a4a..3a3dfc1b 100644 --- a/src/syscall/ztypes_linux_386.go +++ b/src/syscall/ztypes_linux_386.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build 386 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_amd64.go b/src/syscall/ztypes_linux_amd64.go index 3d223fe2..a3831c1e 100644 --- a/src/syscall/ztypes_linux_amd64.go +++ b/src/syscall/ztypes_linux_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build amd64 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_arm.go b/src/syscall/ztypes_linux_arm.go index 9db1142c..f29c8733 100644 --- a/src/syscall/ztypes_linux_arm.go +++ b/src/syscall/ztypes_linux_arm.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build arm && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_arm64.go b/src/syscall/ztypes_linux_arm64.go index 996950f4..05a750bc 100644 --- a/src/syscall/ztypes_linux_arm64.go +++ b/src/syscall/ztypes_linux_arm64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs -- -fsigned-char types_linux.go - -//go:build arm64 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mips.go b/src/syscall/ztypes_linux_mips.go index 4ce84978..dac89f4f 100644 --- a/src/syscall/ztypes_linux_mips.go +++ b/src/syscall/ztypes_linux_mips.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mips64.go b/src/syscall/ztypes_linux_mips64.go index de39e732..b7e55901 100644 --- a/src/syscall/ztypes_linux_mips64.go +++ b/src/syscall/ztypes_linux_mips64.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mips64le.go b/src/syscall/ztypes_linux_mips64le.go index de39e732..b7e55901 100644 --- a/src/syscall/ztypes_linux_mips64le.go +++ b/src/syscall/ztypes_linux_mips64le.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mipsle.go b/src/syscall/ztypes_linux_mipsle.go index 4ce84978..dac89f4f 100644 --- a/src/syscall/ztypes_linux_mipsle.go +++ b/src/syscall/ztypes_linux_mipsle.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_ppc64.go b/src/syscall/ztypes_linux_ppc64.go index 717e2734..74e1435e 100644 --- a/src/syscall/ztypes_linux_ppc64.go +++ b/src/syscall/ztypes_linux_ppc64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build ppc64 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_ppc64le.go b/src/syscall/ztypes_linux_ppc64le.go index 177c1f1e..5828e9b1 100644 --- a/src/syscall/ztypes_linux_ppc64le.go +++ b/src/syscall/ztypes_linux_ppc64le.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build ppc64le && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_riscv64.go b/src/syscall/ztypes_linux_riscv64.go index a6c4d788..1827e5df 100644 --- a/src/syscall/ztypes_linux_riscv64.go +++ b/src/syscall/ztypes_linux_riscv64.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_s390x.go b/src/syscall/ztypes_linux_s390x.go index 5c5a7146..df3414db 100644 --- a/src/syscall/ztypes_linux_s390x.go +++ b/src/syscall/ztypes_linux_s390x.go @@ -1,4 +1,4 @@ -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_386.go b/src/syscall/ztypes_netbsd_386.go index 74eaa4a1..b0e76adb 100644 --- a/src/syscall/ztypes_netbsd_386.go +++ b/src/syscall/ztypes_netbsd_386.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build 386 && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_amd64.go b/src/syscall/ztypes_netbsd_amd64.go index fc28fc9b..68256233 100644 --- a/src/syscall/ztypes_netbsd_amd64.go +++ b/src/syscall/ztypes_netbsd_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build amd64 && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_arm.go b/src/syscall/ztypes_netbsd_arm.go index 1f885048..99ff41e0 100644 --- a/src/syscall/ztypes_netbsd_arm.go +++ b/src/syscall/ztypes_netbsd_arm.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build arm && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_arm64.go b/src/syscall/ztypes_netbsd_arm64.go index cac74693..68256233 100644 --- a/src/syscall/ztypes_netbsd_arm64.go +++ b/src/syscall/ztypes_netbsd_arm64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build arm64 && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_openbsd_386.go b/src/syscall/ztypes_openbsd_386.go index f9ba685e..8dfa22c2 100644 --- a/src/syscall/ztypes_openbsd_386.go +++ b/src/syscall/ztypes_openbsd_386.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_openbsd.go - -//go:build 386 && openbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_openbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_openbsd_amd64.go b/src/syscall/ztypes_openbsd_amd64.go index 889b9551..9382727a 100644 --- a/src/syscall/ztypes_openbsd_amd64.go +++ b/src/syscall/ztypes_openbsd_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_openbsd.go - -//go:build amd64 && openbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_openbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_solaris_amd64.go b/src/syscall/ztypes_solaris_amd64.go index d486cd00..a9f14517 100644 --- a/src/syscall/ztypes_solaris_amd64.go +++ b/src/syscall/ztypes_solaris_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_solaris.go - -//go:build amd64 && solaris +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_solaris.go | go run mkpost.go package syscall diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index 80a1b7de..3a7da9e5 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -5,6 +5,7 @@ package testing import ( + "context" "flag" "fmt" "internal/sysinfo" @@ -78,7 +79,7 @@ type InternalBenchmark struct { } // B is a type passed to [Benchmark] functions to manage benchmark -// timing and to specify the number of iterations to run. +// timing and control the number of iterations. // // A benchmark ends when its Benchmark function returns or calls any of the methods // FailNow, Fatal, Fatalf, SkipNow, Skip, or Skipf. Those methods must be called @@ -93,7 +94,7 @@ type InternalBenchmark struct { type B struct { common importPath string // import path of the package containing the benchmark - context *benchContext + bstate *benchState N int previousN int // number of iterations in the previous run previousDuration time.Duration // total duration of the previous run @@ -113,6 +114,10 @@ type B struct { netBytes uint64 // Extra metrics collected by ReportMetric. extra map[string]float64 + // For Loop() to be executed in benchFunc. + // Loop() has its own control logic that skips the loop scaling. + // See issue #61515. + loopN int } // StartTimer starts timing a test. This function is called automatically @@ -129,8 +134,7 @@ func (b *B) StartTimer() { } // StopTimer stops timing a test. This can be used to pause the timer -// while performing complex initialization that you don't -// want to measure. +// while performing steps that you don't want to measure. func (b *B) StopTimer() { if b.timerOn { b.duration += highPrecisionTimeSince(b.start) @@ -178,6 +182,7 @@ func (b *B) ReportAllocs() { func (b *B) runN(n int) { benchmarkLock.Lock() defer benchmarkLock.Unlock() + ctx, cancelCtx := context.WithCancel(context.Background()) defer func() { b.runCleanup(normalPanic) b.checkRaces() @@ -187,6 +192,10 @@ func (b *B) runN(n int) { runtime.GC() b.resetRaces() b.N = n + b.loopN = 0 + b.ctx = ctx + b.cancelCtx = cancelCtx + b.parallelism = 1 b.ResetTimer() b.StartTimer() @@ -199,10 +208,10 @@ func (b *B) runN(n int) { // run1 runs the first iteration of benchFunc. It reports whether more // iterations of this benchmarks should be run. func (b *B) run1() bool { - if ctx := b.context; ctx != nil { + if bstate := b.bstate; bstate != nil { // Extend maxLen, if needed. - if n := len(b.name) + ctx.extLen + 1; n > ctx.maxLen { - ctx.maxLen = n + 8 // Add additional slack to avoid too many jumps in size. + if n := len(b.name) + bstate.extLen + 1; n > bstate.maxLen { + bstate.maxLen = n + 8 // Add additional slack to avoid too many jumps in size. } } go func() { @@ -253,9 +262,9 @@ func (b *B) run() { fmt.Fprintf(b.w, "cpu: %s\n", cpu) } }) - if b.context != nil { + if b.bstate != nil { // Running go test --test.bench - b.context.processBench(b) // Must call doBench. + b.bstate.processBench(b) // Must call doBench. } else { // Running func Benchmark. b.doBench() @@ -268,6 +277,29 @@ func (b *B) doBench() BenchmarkResult { return b.result } +func predictN(goalns int64, prevIters int64, prevns int64, last int64) int { + if prevns == 0 { + // Round up to dodge divide by zero. See https://go.dev/issue/70709. + prevns = 1 + } + + // Order of operations matters. + // For very fast benchmarks, prevIters ~= prevns. + // If you divide first, you get 0 or 1, + // which can hide an order of magnitude in execution time. + // So multiply first, then divide. + n := goalns * prevIters / prevns + // Run more iterations than we think we'll need (1.2x). + n += n / 5 + // Don't grow too fast in case we had timing errors previously. + n = min(n, 100*last) + // Be sure to run at least one more than last time. + n = max(n, last+1) + // Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.) + n = min(n, 1e9) + return int(n) +} + // launch launches the benchmark function. It gradually increases the number // of benchmark iterations until the benchmark runs for the requested benchtime. // launch is run by the doBench function as a separate goroutine. @@ -279,41 +311,27 @@ func (b *B) launch() { b.signal <- true }() - // Run the benchmark for at least the specified amount of time. - if b.benchTime.n > 0 { - // We already ran a single iteration in run1. - // If -benchtime=1x was requested, use that result. - // See https://golang.org/issue/32051. - if b.benchTime.n > 1 { - b.runN(b.benchTime.n) - } - } else { - d := b.benchTime.d - for n := int64(1); !b.failed && b.duration < d && n < 1e9; { - last := n - // Predict required iterations. - goalns := d.Nanoseconds() - prevIters := int64(b.N) - prevns := b.duration.Nanoseconds() - if prevns <= 0 { - // Round up, to avoid div by zero. - prevns = 1 + // b.Loop does its own ramp-up logic so we just need to run it once. + // If b.loopN is non zero, it means b.Loop has already run. + if b.loopN == 0 { + // Run the benchmark for at least the specified amount of time. + if b.benchTime.n > 0 { + // We already ran a single iteration in run1. + // If -benchtime=1x was requested, use that result. + // See https://golang.org/issue/32051. + if b.benchTime.n > 1 { + b.runN(b.benchTime.n) + } + } else { + d := b.benchTime.d + for n := int64(1); !b.failed && b.duration < d && n < 1e9; { + last := n + // Predict required iterations. + goalns := d.Nanoseconds() + prevIters := int64(b.N) + n = int64(predictN(goalns, prevIters, b.duration.Nanoseconds(), last)) + b.runN(int(n)) } - // Order of operations matters. - // For very fast benchmarks, prevIters ~= prevns. - // If you divide first, you get 0 or 1, - // which can hide an order of magnitude in execution time. - // So multiply first, then divide. - n = goalns * prevIters / prevns - // Run more iterations than we think we'll need (1.2x). - n += n / 5 - // Don't grow too fast in case we had timing errors previously. - n = min(n, 100*last) - // Be sure to run at least one more than last time. - n = max(n, last+1) - // Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.) - n = min(n, 1e9) - b.runN(int(n)) } } b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes, b.extra} @@ -349,6 +367,86 @@ func (b *B) ReportMetric(n float64, unit string) { b.extra[unit] = n } +func (b *B) stopOrScaleBLoop() bool { + timeElapsed := highPrecisionTimeSince(b.start) + if timeElapsed >= b.benchTime.d { + // Stop the timer so we don't count cleanup time + b.StopTimer() + return false + } + // Loop scaling + goalns := b.benchTime.d.Nanoseconds() + prevIters := int64(b.N) + b.N = predictN(goalns, prevIters, timeElapsed.Nanoseconds(), prevIters) + b.loopN++ + return true +} + +func (b *B) loopSlowPath() bool { + if b.loopN == 0 { + // If it's the first call to b.Loop() in the benchmark function. + // Allows more precise measurement of benchmark loop cost counts. + // Also initialize b.N to 1 to kick start loop scaling. + b.N = 1 + b.loopN = 1 + b.ResetTimer() + return true + } + // Handles fixed iterations case + if b.benchTime.n > 0 { + if b.N < b.benchTime.n { + b.N = b.benchTime.n + b.loopN++ + return true + } + b.StopTimer() + return false + } + // Handles fixed time case + return b.stopOrScaleBLoop() +} + +// Loop returns true as long as the benchmark should continue running. +// +// A typical benchmark is structured like: +// +// func Benchmark(b *testing.B) { +// ... setup ... +// for b.Loop() { +// ... code to measure ... +// } +// ... cleanup ... +// } +// +// Loop resets the benchmark timer the first time it is called in a benchmark, +// so any setup performed prior to starting the benchmark loop does not count +// toward the benchmark measurement. Likewise, when it returns false, it stops +// the timer so cleanup code is not measured. +// +// The compiler never optimizes away calls to functions within the body of a +// "for b.Loop() { ... }" loop. This prevents surprises that can otherwise occur +// if the compiler determines that the result of a benchmarked function is +// unused. The loop must be written in exactly this form, and this only applies +// to calls syntactically between the curly braces of the loop. Optimizations +// are performed as usual in any functions called by the loop. +// +// After Loop returns false, b.N contains the total number of iterations that +// ran, so the benchmark may use b.N to compute other average metrics. +// +// Prior to the introduction of Loop, benchmarks were expected to contain an +// explicit loop from 0 to b.N. Benchmarks should either use Loop or contain a +// loop to b.N, but not both. Loop offers more automatic management of the +// benchmark timer, and runs each benchmark function only once per measurement, +// whereas b.N-based benchmarks must run the benchmark function (and any +// associated setup and cleanup) several times. +func (b *B) Loop() bool { + if b.loopN != 0 && b.loopN < b.N { + b.loopN++ + return true + } + return b.loopSlowPath() +} + // BenchmarkResult contains the results of a benchmark run. type BenchmarkResult struct { N int // The number of iterations. @@ -492,7 +590,7 @@ func benchmarkName(name string, n int) string { return name } -type benchContext struct { +type benchState struct { match *matcher maxLen int // The largest recorded benchmark name. @@ -517,17 +615,17 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e maxprocs = procs } } - ctx := &benchContext{ + bstate := &benchState{ match: newMatcher(matchString, *matchBenchmarks, "-test.bench", *skip), extLen: len(benchmarkName("", maxprocs)), } var bs []InternalBenchmark for _, Benchmark := range benchmarks { - if _, matched, _ := ctx.match.fullName(nil, Benchmark.Name); matched { + if _, matched, _ := bstate.match.fullName(nil, Benchmark.Name); matched { bs = append(bs, Benchmark) benchName := benchmarkName(Benchmark.Name, maxprocs) - if l := len(benchName) + ctx.extLen + 1; l > ctx.maxLen { - ctx.maxLen = l + if l := len(benchName) + bstate.extLen + 1; l > bstate.maxLen { + bstate.maxLen = l } } } @@ -544,7 +642,7 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e } }, benchTime: benchTime, - context: ctx, + bstate: bstate, } if Verbose() { main.chatty = newChattyPrinter(main.w) @@ -554,7 +652,7 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e } // processBench runs bench b for the configured CPU counts and prints the results. -func (ctx *benchContext) processBench(b *B) { +func (s *benchState) processBench(b *B) { for i, procs := range cpuList { for j := uint(0); j < *count; j++ { runtime.GOMAXPROCS(procs) @@ -562,7 +660,7 @@ func (ctx *benchContext) processBench(b *B) { // If it's chatty, we've already printed this information. if b.chatty == nil { - fmt.Fprintf(b.w, "%-*s\t", ctx.maxLen, benchName) + fmt.Fprintf(b.w, "%-*s\t", s.maxLen, benchName) } // Recompute the running time for all but the first iteration. if i > 0 || j > 0 { @@ -589,7 +687,7 @@ func (ctx *benchContext) processBench(b *B) { } results := r.String() if b.chatty != nil { - fmt.Fprintf(b.w, "%-*s\t", ctx.maxLen, benchName) + fmt.Fprintf(b.w, "%-*s\t", s.maxLen, benchName) } if *benchmarkMemory || b.showAllocResult { results += "\t" + r.MemString() @@ -629,8 +727,8 @@ func (b *B) Run(name string, f func(b *B)) bool { defer benchmarkLock.Lock() benchName, ok, partial := b.name, true, false - if b.context != nil { - benchName, ok, partial = b.context.match.fullName(&b.common, name) + if b.bstate != nil { + benchName, ok, partial = b.bstate.match.fullName(&b.common, name) } if !ok { return true @@ -651,7 +749,7 @@ func (b *B) Run(name string, f func(b *B)) bool { importPath: b.importPath, benchFunc: f, benchTime: b.benchTime, - context: b.context, + bstate: b.bstate, } if partial { // Partial name match, like -bench=X/Y matching BenchmarkX. diff --git a/src/testing/benchmark_test.go b/src/testing/benchmark_test.go index 66f555d1..e2dd24c8 100644 --- a/src/testing/benchmark_test.go +++ b/src/testing/benchmark_test.go @@ -7,6 +7,8 @@ package testing_test import ( "bytes" "cmp" + "context" + "errors" "runtime" "slices" "strings" @@ -127,6 +129,34 @@ func TestRunParallelSkipNow(t *testing.T) { }) } +func TestBenchmarkContext(t *testing.T) { + testing.Benchmark(func(b *testing.B) { + ctx := b.Context() + if err := ctx.Err(); err != nil { + b.Fatalf("expected non-canceled context, got %v", err) + } + + var innerCtx context.Context + b.Run("inner", func(b *testing.B) { + innerCtx = b.Context() + if err := innerCtx.Err(); err != nil { + b.Fatalf("expected inner benchmark to not inherit canceled context, got %v", err) + } + }) + b.Run("inner2", func(b *testing.B) { + if !errors.Is(innerCtx.Err(), context.Canceled) { + t.Fatal("expected context of sibling benchmark to be canceled after its test function finished") + } + }) + + t.Cleanup(func() { + if !errors.Is(ctx.Err(), context.Canceled) { + t.Fatal("expected context canceled before cleanup") + } + }) + }) +} + func ExampleB_RunParallel() { // Parallel benchmark for text/template.Template.Execute on a single object. testing.Benchmark(func(b *testing.B) { @@ -167,7 +197,7 @@ func ExampleB_ReportMetric() { // specific algorithm (in this case, sorting). testing.Benchmark(func(b *testing.B) { var compares int64 - for i := 0; i < b.N; i++ { + for b.Loop() { s := []int{5, 4, 3, 2, 1} slices.SortFunc(s, func(a, b int) int { compares++ diff --git a/src/testing/example.go b/src/testing/example.go index b14477a4..c343ae2a 100644 --- a/src/testing/example.go +++ b/src/testing/example.go @@ -6,6 +6,7 @@ package testing import ( "fmt" + "runtime" "slices" "strings" "time" @@ -66,6 +67,10 @@ func (eg *InternalExample) processRunResult(stdout string, timeSpent time.Durati var fail string got := strings.TrimSpace(stdout) want := strings.TrimSpace(eg.Output) + if runtime.GOOS == "windows" { + got = strings.ReplaceAll(got, "\r\n", "\n") + want = strings.ReplaceAll(want, "\r\n", "\n") + } if eg.Unordered { if sortLines(got) != sortLines(want) && recovered == nil { fail = fmt.Sprintf("got:\n%s\nwant (unordered):\n%s\n", stdout, eg.Output) diff --git a/src/testing/example_loop_test.go b/src/testing/example_loop_test.go new file mode 100644 index 00000000..eff8bab3 --- /dev/null +++ b/src/testing/example_loop_test.go @@ -0,0 +1,48 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing_test + +import ( + "math/rand/v2" + "testing" +) + +// ExBenchmark shows how to use b.Loop in a benchmark. +// +// (If this were a real benchmark, not an example, this would be named +// BenchmarkSomething.) +func ExBenchmark(b *testing.B) { + // Generate a large random slice to use as an input. + // Since this is done before the first call to b.Loop(), + // it doesn't count toward the benchmark time. + input := make([]int, 128<<10) + for i := range input { + input[i] = rand.Int() + } + + // Perform the benchmark. + for b.Loop() { + // Normally, the compiler would be allowed to optimize away the call + // to sum because it has no side effects and the result isn't used. + // However, inside a b.Loop loop, the compiler ensures function calls + // aren't optimized away. + sum(input) + } + + // Outside the loop, the timer is stopped, so we could perform + // cleanup if necessary without affecting the result. +} + +func sum(data []int) int { + total := 0 + for _, value := range data { + total += value + } + return total +} + +func ExampleB_Loop() { + testing.Benchmark(ExBenchmark) +} diff --git a/src/testing/export_test.go b/src/testing/export_test.go index 10a5b04a..a2dddc79 100644 --- a/src/testing/export_test.go +++ b/src/testing/export_test.go @@ -9,3 +9,5 @@ var PrettyPrint = prettyPrint type HighPrecisionTime = highPrecisionTime var HighPrecisionTimeNow = highPrecisionTimeNow + +const ParallelConflict = parallelConflict diff --git a/src/testing/flag_test.go b/src/testing/flag_test.go index 6f76c237..6a775442 100644 --- a/src/testing/flag_test.go +++ b/src/testing/flag_test.go @@ -28,11 +28,7 @@ func TestFlag(t *testing.T) { flag := flag t.Run(flag, func(t *testing.T) { t.Parallel() - exe, err := os.Executable() - if err != nil { - exe = os.Args[0] - } - cmd := exec.Command(exe, "-test.run=^TestFlag$", "-test_flag_arg="+flag) + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestFlag$", "-test_flag_arg="+flag) if flag != "" { cmd.Args = append(cmd.Args, flag) } diff --git a/src/testing/fstest/testfs.go b/src/testing/fstest/testfs.go index 080bcdd6..affdfa64 100644 --- a/src/testing/fstest/testfs.go +++ b/src/testing/fstest/testfs.go @@ -10,8 +10,8 @@ import ( "fmt" "io" "io/fs" + "maps" "path" - "reflect" "slices" "strings" "testing/iotest" @@ -72,13 +72,7 @@ func testFS(fsys fs.FS, expected ...string) error { } delete(found, ".") if len(expected) == 0 && len(found) > 0 { - var list []string - for k := range found { - if k != "." { - list = append(list, k) - } - } - slices.Sort(list) + list := slices.Sorted(maps.Keys(found)) if len(list) > 15 { list = append(list[:10], "...") } @@ -358,7 +352,7 @@ func (t *fsTester) checkGlob(dir string, list []fs.DirEntry) { t.errorf("%s: Glob(%#q): %w", dir, glob, err) return } - if reflect.DeepEqual(want, names) { + if slices.Equal(want, names) { return } @@ -576,7 +570,7 @@ func (t *fsTester) checkFileRead(file, desc string, data1, data2 []byte) { } } -// checkBadPath checks that various invalid forms of file's name cannot be opened using t.fsys.Open. +// checkOpen validates file opening behavior by attempting to open and then close the given file path. func (t *fsTester) checkOpen(file string) { t.checkBadPath(file, "Open", func(file string) error { f, err := t.fsys.Open(file) diff --git a/src/testing/fuzz.go b/src/testing/fuzz.go index d561225b..dceb786a 100644 --- a/src/testing/fuzz.go +++ b/src/testing/fuzz.go @@ -5,6 +5,7 @@ package testing import ( + "context" "errors" "flag" "fmt" @@ -67,8 +68,8 @@ type InternalFuzzTarget struct { // that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name. type F struct { common - fuzzContext *fuzzContext - testContext *testContext + fstate *fuzzState + tstate *testState // inFuzzFn is true when the fuzz function is running. Most F methods cannot // be called when inFuzzFn is true. @@ -244,22 +245,22 @@ func (f *F) Fuzz(ff any) { // corpus and entries declared with F.Add. // // Don't load the seed corpus if this is a worker process; we won't use it. - if f.fuzzContext.mode != fuzzWorker { + if f.fstate.mode != fuzzWorker { for _, c := range f.corpus { - if err := f.fuzzContext.deps.CheckCorpus(c.Values, types); err != nil { + if err := f.fstate.deps.CheckCorpus(c.Values, types); err != nil { // TODO(#48302): Report the source location of the F.Add call. f.Fatal(err) } } // Load seed corpus - c, err := f.fuzzContext.deps.ReadCorpus(filepath.Join(corpusDir, f.name), types) + c, err := f.fstate.deps.ReadCorpus(filepath.Join(corpusDir, f.name), types) if err != nil { f.Fatal(err) } for i := range c { c[i].IsSeed = true // these are all seed corpus values - if f.fuzzContext.mode == fuzzCoordinator { + if f.fstate.mode == fuzzCoordinator { // If this is the coordinator process, zero the values, since we don't need // to hold onto them. c[i].Values = nil @@ -285,14 +286,16 @@ func (f *F) Fuzz(ff any) { if e.Path != "" { testName = fmt.Sprintf("%s/%s", testName, filepath.Base(e.Path)) } - if f.testContext.isFuzzing { + if f.tstate.isFuzzing { // Don't preserve subtest names while fuzzing. If fn calls T.Run, // there will be a very large number of subtests with duplicate names, // which will use a large amount of memory. The subtest names aren't // useful since there's no way to re-run them deterministically. - f.testContext.match.clearSubNames() + f.tstate.match.clearSubNames() } + ctx, cancelCtx := context.WithCancel(f.ctx) + // Record the stack trace at the point of this call so that if the subtest // function - which runs in a separate stack - is marked as a helper, we can // continue walking the stack into the parent test. @@ -300,15 +303,17 @@ func (f *F) Fuzz(ff any) { n := runtime.Callers(2, pc[:]) t := &T{ common: common{ - barrier: make(chan bool), - signal: make(chan bool), - name: testName, - parent: &f.common, - level: f.level + 1, - creator: pc[:n], - chatty: f.chatty, + barrier: make(chan bool), + signal: make(chan bool), + name: testName, + parent: &f.common, + level: f.level + 1, + creator: pc[:n], + chatty: f.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - context: f.testContext, + tstate: f.tstate, } if captureOut != nil { // t.parent aliases f.common. @@ -328,9 +333,9 @@ func (f *F) Fuzz(ff any) { // we make sure it is called right before the tRunner function // exits, regardless of whether it was executed cleanly, panicked, // or if the fuzzFn called t.Fatal. - if f.testContext.isFuzzing { - defer f.fuzzContext.deps.SnapshotCoverage() - f.fuzzContext.deps.ResetCoverage() + if f.tstate.isFuzzing { + defer f.fstate.deps.SnapshotCoverage() + f.fstate.deps.ResetCoverage() } fn.Call(args) }) @@ -342,14 +347,14 @@ func (f *F) Fuzz(ff any) { return !t.Failed() } - switch f.fuzzContext.mode { + switch f.fstate.mode { case fuzzCoordinator: // Fuzzing is enabled, and this is the test process started by 'go test'. // Act as the coordinator process, and coordinate workers to perform the // actual fuzzing. corpusTargetDir := filepath.Join(corpusDir, f.name) cacheTargetDir := filepath.Join(*fuzzCacheDir, f.name) - err := f.fuzzContext.deps.CoordinateFuzzing( + err := f.fstate.deps.CoordinateFuzzing( fuzzDuration.d, int64(fuzzDuration.n), minimizeDuration.d, @@ -376,7 +381,7 @@ func (f *F) Fuzz(ff any) { case fuzzWorker: // Fuzzing is enabled, and this is a worker process. Follow instructions // from the coordinator. - if err := f.fuzzContext.deps.RunFuzzWorker(func(e corpusEntry) error { + if err := f.fstate.deps.RunFuzzWorker(func(e corpusEntry) error { // Don't write to f.w (which points to Stdout) if running from a // fuzz worker. This would become very verbose, particularly during // minimization. Return the error instead, and let the caller deal @@ -398,7 +403,7 @@ func (f *F) Fuzz(ff any) { // corpus now. for _, e := range f.corpus { name := fmt.Sprintf("%s/%s", f.name, filepath.Base(e.Path)) - if _, ok, _ := f.testContext.match.fullName(nil, name); ok { + if _, ok, _ := f.tstate.match.fullName(nil, name); ok { run(f.w, e) } } @@ -451,8 +456,8 @@ type fuzzCrashError interface { CrashPath() string } -// fuzzContext holds fields common to all fuzz tests. -type fuzzContext struct { +// fuzzState holds fields common to all fuzz tests. +type fuzzState struct { deps testDeps mode fuzzMode } @@ -486,9 +491,9 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T break } - tctx := newTestContext(*parallel, m) - tctx.deadline = deadline - fctx := &fuzzContext{deps: deps, mode: seedCorpusOnly} + tstate := newTestState(*parallel, m) + tstate.deadline = deadline + fstate := &fuzzState{deps: deps, mode: seedCorpusOnly} root := common{w: os.Stdout} // gather output in one place if Verbose() { root.chatty = newChattyPrinter(root.w) @@ -497,7 +502,7 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T if shouldFailFast() { break } - testName, matched, _ := tctx.match.fullName(nil, ft.Name) + testName, matched, _ := tstate.match.fullName(nil, ft.Name) if !matched { continue } @@ -508,17 +513,20 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T continue } } + ctx, cancelCtx := context.WithCancel(context.Background()) f := &F{ common: common{ - signal: make(chan bool), - barrier: make(chan bool), - name: testName, - parent: &root, - level: root.level + 1, - chatty: root.chatty, + signal: make(chan bool), + barrier: make(chan bool), + name: testName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - testContext: tctx, - fuzzContext: fctx, + tstate: tstate, + fstate: fstate, } f.w = indenter{&f.common} if f.chatty != nil { @@ -554,17 +562,17 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) { return true } m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip) - tctx := newTestContext(1, m) - tctx.isFuzzing = true - fctx := &fuzzContext{ + tstate := newTestState(1, m) + tstate.isFuzzing = true + fstate := &fuzzState{ deps: deps, } root := common{w: os.Stdout} if *isFuzzWorker { root.w = io.Discard - fctx.mode = fuzzWorker + fstate.mode = fuzzWorker } else { - fctx.mode = fuzzCoordinator + fstate.mode = fuzzCoordinator } if Verbose() && !*isFuzzWorker { root.chatty = newChattyPrinter(root.w) @@ -573,7 +581,7 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) { var testName string var matched []string for i := range fuzzTests { - name, ok, _ := tctx.match.fullName(nil, fuzzTests[i].Name) + name, ok, _ := tstate.match.fullName(nil, fuzzTests[i].Name) if !ok { continue } @@ -590,17 +598,20 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) { return false } + ctx, cancelCtx := context.WithCancel(context.Background()) f := &F{ common: common{ - signal: make(chan bool), - barrier: nil, // T.Parallel has no effect when fuzzing. - name: testName, - parent: &root, - level: root.level + 1, - chatty: root.chatty, + signal: make(chan bool), + barrier: nil, // T.Parallel has no effect when fuzzing. + name: testName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - fuzzContext: fctx, - testContext: tctx, + fstate: fstate, + tstate: tstate, } f.w = indenter{&f.common} if f.chatty != nil { @@ -694,7 +705,7 @@ func fRunner(f *F, fn func(*F)) { // This only affects fuzz tests run as normal tests. // While fuzzing, T.Parallel has no effect, so f.sub is empty, and this // branch is not taken. f.barrier is nil in that case. - f.testContext.release() + f.tstate.release() close(f.barrier) // Wait for the subtests to complete. for _, sub := range f.sub { diff --git a/src/testing/helper_test.go b/src/testing/helper_test.go index da5622f8..a698e79f 100644 --- a/src/testing/helper_test.go +++ b/src/testing/helper_test.go @@ -23,15 +23,9 @@ func TestTBHelper(t *testing.T) { return } - testenv.MustHaveExec(t) t.Parallel() - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - - cmd := testenv.Command(t, exe, "-test.run=^TestTBHelper$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestTBHelper$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") out, _ := cmd.CombinedOutput() @@ -66,15 +60,9 @@ func TestTBHelperParallel(t *testing.T) { return } - testenv.MustHaveExec(t) t.Parallel() - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - - cmd := testenv.Command(t, exe, "-test.run=^TestTBHelperParallel$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestTBHelperParallel$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") out, _ := cmd.CombinedOutput() diff --git a/src/testing/loop_test.go b/src/testing/loop_test.go new file mode 100644 index 00000000..7a1a93fc --- /dev/null +++ b/src/testing/loop_test.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +func TestBenchmarkBLoop(t *T) { + var initialStart highPrecisionTime + var firstStart highPrecisionTime + var lastStart highPrecisionTime + var runningEnd bool + runs := 0 + iters := 0 + finalBN := 0 + bRet := Benchmark(func(b *B) { + initialStart = b.start + runs++ + for b.Loop() { + if iters == 0 { + firstStart = b.start + } + lastStart = b.start + iters++ + } + finalBN = b.N + runningEnd = b.timerOn + }) + // Verify that a b.Loop benchmark is invoked just once. + if runs != 1 { + t.Errorf("want runs == 1, got %d", runs) + } + // Verify that at least one iteration ran. + if iters == 0 { + t.Fatalf("no iterations ran") + } + // Verify that b.N, bRet.N, and the b.Loop() iteration count match. + if finalBN != iters || bRet.N != iters { + t.Errorf("benchmark iterations mismatch: %d loop iterations, final b.N=%d, bRet.N=%d", iters, finalBN, bRet.N) + } + // Make sure the benchmark ran for an appropriate amount of time. + if bRet.T < benchTime.d { + t.Fatalf("benchmark ran for %s, want >= %s", bRet.T, benchTime.d) + } + // Verify that the timer is reset on the first loop, and then left alone. + if firstStart == initialStart { + t.Errorf("b.Loop did not reset the timer") + } + if lastStart != firstStart { + t.Errorf("timer was reset during iteration") + } + // Verify that it stopped the timer after the last loop. + if runningEnd { + t.Errorf("timer was still running after last iteration") + } +} + +// See also TestBenchmarkBLoop* in other files. diff --git a/src/testing/slogtest/slogtest.go b/src/testing/slogtest/slogtest.go index 5c3aced6..f9e2d900 100644 --- a/src/testing/slogtest/slogtest.go +++ b/src/testing/slogtest/slogtest.go @@ -265,7 +265,7 @@ func TestHandler(h slog.Handler, results func() []map[string]any) error { if g, w := len(res), len(cases); g != w { return fmt.Errorf("got %d results, want %d", g, w) } - for i, got := range results() { + for i, got := range res { c := cases[i] for _, check := range c.checks { if problem := check(got); problem != "" { diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 1c23d054..82ec5809 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -7,9 +7,9 @@ package testing import ( "bytes" "fmt" - "reflect" "regexp" "runtime" + "slices" "strings" "sync" "sync/atomic" @@ -21,12 +21,11 @@ func init() { benchTime.d = 100 * time.Millisecond } -func TestTestContext(t *T) { +func TestTestState(t *T) { const ( add1 = 0 done = 1 ) - // After each of the calls are applied to the context, the type call struct { typ int // run or done // result from applying the call @@ -72,7 +71,7 @@ func TestTestContext(t *T) { }, }} for i, tc := range testCases { - ctx := &testContext{ + tstate := &testState{ startParallel: make(chan bool), maxParallel: tc.max, } @@ -88,18 +87,18 @@ func TestTestContext(t *T) { started := false switch call.typ { case add1: - signal := doCall(ctx.waitParallel) + signal := doCall(tstate.waitParallel) select { case <-signal: started = true - case ctx.startParallel <- true: + case tstate.startParallel <- true: <-signal } case done: - signal := doCall(ctx.release) + signal := doCall(tstate.release) select { case <-signal: - case <-ctx.startParallel: + case <-tstate.startParallel: started = true <-signal } @@ -107,11 +106,11 @@ func TestTestContext(t *T) { if started != call.started { t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started) } - if ctx.running != call.running { - t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running) + if tstate.running != call.running { + t.Errorf("%d:%d:running: got %v; want %v", i, j, tstate.running, call.running) } - if ctx.numWaiting != call.waiting { - t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting) + if tstate.numWaiting != call.waiting { + t.Errorf("%d:%d:waiting: got %v; want %v", i, j, tstate.numWaiting, call.waiting) } } } @@ -507,7 +506,7 @@ func TestTRun(t *T) { }} for _, tc := range testCases { t.Run(tc.desc, func(t *T) { - ctx := newTestContext(tc.maxPar, allMatcher()) + tstate := newTestState(tc.maxPar, allMatcher()) buf := &strings.Builder{} root := &T{ common: common{ @@ -516,14 +515,14 @@ func TestTRun(t *T) { name: "", w: buf, }, - context: ctx, + tstate: tstate, } if tc.chatty { root.chatty = newChattyPrinter(root.w) root.chatty.json = tc.json } ok := root.Run(tc.desc, tc.f) - ctx.release() + tstate.release() if ok != tc.ok { t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok) @@ -531,8 +530,8 @@ func TestTRun(t *T) { if ok != !root.Failed() { t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed()) } - if ctx.running != 0 || ctx.numWaiting != 0 { - t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting) + if tstate.running != 0 || tstate.numWaiting != 0 { + t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, tstate.running, tstate.numWaiting) } got := strings.TrimSpace(buf.String()) want := strings.TrimSpace(tc.output) @@ -790,8 +789,8 @@ func TestRacyOutput(t *T) { } root := &T{ - common: common{w: &funcWriter{raceDetector}}, - context: newTestContext(1, allMatcher()), + common: common{w: &funcWriter{raceDetector}}, + tstate: newTestState(1, allMatcher()), } root.chatty = newChattyPrinter(root.w) root.Run("", func(t *T) { @@ -815,7 +814,7 @@ func TestRacyOutput(t *T) { // The late log message did not include the test name. Issue 29388. func TestLogAfterComplete(t *T) { - ctx := newTestContext(1, allMatcher()) + tstate := newTestState(1, allMatcher()) var buf bytes.Buffer t1 := &T{ common: common{ @@ -824,7 +823,7 @@ func TestLogAfterComplete(t *T) { signal: make(chan bool, 1), w: &buf, }, - context: ctx, + tstate: tstate, } c1 := make(chan bool) @@ -886,7 +885,7 @@ func TestCleanup(t *T) { t.Cleanup(func() { cleanups = append(cleanups, 1) }) t.Cleanup(func() { cleanups = append(cleanups, 2) }) }) - if got, want := cleanups, []int{2, 1}; !reflect.DeepEqual(got, want) { + if got, want := cleanups, []int{2, 1}; !slices.Equal(got, want) { t.Errorf("unexpected cleanup record; got %v want %v", got, want) } } diff --git a/src/testing/synctest/context_example_test.go b/src/testing/synctest/context_example_test.go new file mode 100644 index 00000000..5f7205e5 --- /dev/null +++ b/src/testing/synctest/context_example_test.go @@ -0,0 +1,78 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.synctest + +package synctest_test + +import ( + "context" + "fmt" + "testing/synctest" + "time" +) + +// This example demonstrates testing the context.AfterFunc function. +// +// AfterFunc registers a function to execute in a new goroutine +// after a context is canceled. +// +// The test verifies that the function is not run before the context is canceled, +// and is run after the context is canceled. +func Example_contextAfterFunc() { + synctest.Run(func() { + // Create a context.Context which can be canceled. + ctx, cancel := context.WithCancel(context.Background()) + + // context.AfterFunc registers a function to be called + // when a context is canceled. + afterFuncCalled := false + context.AfterFunc(ctx, func() { + afterFuncCalled = true + }) + + // The context has not been canceled, so the AfterFunc is not called. + synctest.Wait() + fmt.Printf("before context is canceled: afterFuncCalled=%v\n", afterFuncCalled) + + // Cancel the context and wait for the AfterFunc to finish executing. + // Verify that the AfterFunc ran. + cancel() + synctest.Wait() + fmt.Printf("after context is canceled: afterFuncCalled=%v\n", afterFuncCalled) + + // Output: + // before context is canceled: afterFuncCalled=false + // after context is canceled: afterFuncCalled=true + }) +} + +// This example demonstrates testing the context.WithTimeout function. +// +// WithTimeout creates a context which is canceled after a timeout. +// +// The test verifies that the context is not canceled before the timeout expires, +// and is canceled after the timeout expires. +func Example_contextWithTimeout() { + synctest.Run(func() { + // Create a context.Context which is canceled after a timeout. + const timeout = 5 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // Wait just less than the timeout. + time.Sleep(timeout - time.Nanosecond) + synctest.Wait() + fmt.Printf("before timeout: ctx.Err() = %v\n", ctx.Err()) + + // Wait the rest of the way until the timeout. + time.Sleep(time.Nanosecond) + synctest.Wait() + fmt.Printf("after timeout: ctx.Err() = %v\n", ctx.Err()) + + // Output: + // before timeout: ctx.Err() = + // after timeout: ctx.Err() = context deadline exceeded + }) +} diff --git a/src/testing/synctest/synctest.go b/src/testing/synctest/synctest.go new file mode 100644 index 00000000..90efc789 --- /dev/null +++ b/src/testing/synctest/synctest.go @@ -0,0 +1,67 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.synctest + +// Package synctest provides support for testing concurrent code. +// +// This package only exists when using Go compiled with GOEXPERIMENT=synctest. +// It is experimental, and not subject to the Go 1 compatibility promise. +package synctest + +import ( + "internal/synctest" +) + +// Run executes f in a new goroutine. +// +// The new goroutine and any goroutines transitively started by it form +// an isolated "bubble". +// Run waits for all goroutines in the bubble to exit before returning. +// +// Goroutines in the bubble use a synthetic time implementation. +// The initial time is midnight UTC 2000-01-01. +// +// Time advances when every goroutine in the bubble is blocked. +// For example, a call to time.Sleep will block until all other +// goroutines are blocked and return after the bubble's clock has +// advanced. See [Wait] for the specific definition of blocked. +// +// If every goroutine is blocked and there are no timers scheduled, +// Run panics. +// +// Channels, time.Timers, and time.Tickers created within the bubble +// are associated with it. Operating on a bubbled channel, timer, or ticker +// from outside the bubble panics. +func Run(f func()) { + synctest.Run(f) +} + +// Wait blocks until every goroutine within the current bubble, +// other than the current goroutine, is durably blocked. +// It panics if called from a non-bubbled goroutine, +// or if two goroutines in the same bubble call Wait at the same time. +// +// A goroutine is durably blocked if can only be unblocked by another +// goroutine in its bubble. The following operations durably block +// a goroutine: +// - a send or receive on a channel from within the bubble +// - a select statement where every case is a channel within the bubble +// - sync.Cond.Wait +// - time.Sleep +// +// A goroutine executing a system call or waiting for an external event +// such as a network operation is not durably blocked. +// For example, a goroutine blocked reading from an network connection +// is not durably blocked even if no data is currently available on the +// connection, because it may be unblocked by data written from outside +// the bubble or may be in the process of receiving data from a kernel +// network buffer. +// +// A goroutine is not durably blocked when blocked on a send or receive +// on a channel that was not created within its bubble, because it may +// be unblocked by a channel receive or send from outside its bubble. +func Wait() { + synctest.Wait() +} diff --git a/src/testing/testing.go b/src/testing/testing.go index 526cba39..3833bfc8 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -72,27 +72,24 @@ // A sample benchmark function looks like this: // // func BenchmarkRandInt(b *testing.B) { -// for range b.N { +// for b.Loop() { // rand.Int() // } // } // -// The benchmark function must run the target code b.N times. -// It is called multiple times with b.N adjusted until the -// benchmark function lasts long enough to be timed reliably. // The output // // BenchmarkRandInt-8 68453040 17.8 ns/op // -// means that the loop ran 68453040 times at a speed of 17.8 ns per loop. +// means that the body of the loop ran 68453040 times at a speed of 17.8 ns per loop. // -// If a benchmark needs some expensive setup before running, the timer -// may be reset: +// Only the body of the loop is timed, so benchmarks may do expensive +// setup before calling b.Loop, which will not be counted toward the +// benchmark measurement: // // func BenchmarkBigLen(b *testing.B) { // big := NewBig() -// b.ResetTimer() -// for range b.N { +// for b.Loop() { // big.Len() // } // } @@ -120,6 +117,37 @@ // In particular, https://golang.org/x/perf/cmd/benchstat performs // statistically robust A/B comparisons. // +// # b.N-style benchmarks +// +// Prior to the introduction of [B.Loop], benchmarks were written in a +// different style using B.N. For example: +// +// func BenchmarkRandInt(b *testing.B) { +// for range b.N { +// rand.Int() +// } +// } +// +// In this style of benchmark, the benchmark function must run +// the target code b.N times. The benchmark function is called +// multiple times with b.N adjusted until the benchmark function +// lasts long enough to be timed reliably. This also means any setup +// done before the loop may be run several times. +// +// If a benchmark needs some expensive setup before running, the timer +// should be explicitly reset: +// +// func BenchmarkBigLen(b *testing.B) { +// big := NewBig() +// b.ResetTimer() +// for range b.N { +// big.Len() +// } +// } +// +// New benchmarks should prefer using [B.Loop], which is more robust +// and more efficient. +// // # Examples // // The package also runs and verifies example code. Example functions may @@ -371,6 +399,7 @@ package testing import ( "bytes" + "context" "errors" "flag" "fmt" @@ -379,6 +408,7 @@ import ( "io" "math/rand" "os" + "path/filepath" "reflect" "runtime" "runtime/debug" @@ -632,6 +662,9 @@ type common struct { tempDir string tempDirErr error tempDirSeq int32 + + ctx context.Context + cancelCtx context.CancelFunc } // Short reports whether the -test.short flag is set. @@ -891,11 +924,13 @@ type TB interface { Logf(format string, args ...any) Name() string Setenv(key, value string) + Chdir(dir string) Skip(args ...any) SkipNow() Skipf(format string, args ...any) Skipped() bool TempDir() string + Context() context.Context // A private method to prevent users implementing the // interface and so future additions to it will not @@ -917,8 +952,8 @@ var _ TB = (*B)(nil) // may be called simultaneously from multiple goroutines. type T struct { common - isEnvSet bool - context *testContext // For running tests and subtests. + denyParallel bool + tstate *testState // For running tests and subtests. } func (c *common) private() {} @@ -1307,6 +1342,58 @@ func (c *common) Setenv(key, value string) { } } +// Chdir calls os.Chdir(dir) and uses Cleanup to restore the current +// working directory to its original value after the test. On Unix, it +// also sets PWD environment variable for the duration of the test. +// +// Because Chdir affects the whole process, it cannot be used +// in parallel tests or tests with parallel ancestors. +func (c *common) Chdir(dir string) { + c.checkFuzzFn("Chdir") + oldwd, err := os.Open(".") + if err != nil { + c.Fatal(err) + } + if err := os.Chdir(dir); err != nil { + c.Fatal(err) + } + // On POSIX platforms, PWD represents “an absolute pathname of the + // current working directory.” Since we are changing the working + // directory, we should also set or update PWD to reflect that. + switch runtime.GOOS { + case "windows", "plan9": + // Windows and Plan 9 do not use the PWD variable. + default: + if !filepath.IsAbs(dir) { + dir, err = os.Getwd() + if err != nil { + c.Fatal(err) + } + } + c.Setenv("PWD", dir) + } + c.Cleanup(func() { + err := oldwd.Chdir() + oldwd.Close() + if err != nil { + // It's not safe to continue with tests if we can't + // get back to the original working directory. Since + // we are holding a dirfd, this is highly unlikely. + panic("testing.Chdir: " + err.Error()) + } + }) +} + +// Context returns a context that is canceled just before +// Cleanup-registered functions are called. +// +// Cleanup functions can wait for any resources +// that shut down on Context.Done before the test or benchmark completes. +func (c *common) Context() context.Context { + c.checkFuzzFn("Context") + return c.ctx +} + // panicHandling controls the panic handling used by runCleanup. type panicHandling int @@ -1339,6 +1426,10 @@ func (c *common) runCleanup(ph panicHandling) (panicVal any) { } }() + if c.cancelCtx != nil { + c.cancelCtx() + } + for { var cleanup func() c.mu.Lock() @@ -1436,6 +1527,8 @@ func pcToName(pc uintptr) string { return frame.Function } +const parallelConflict = `testing: test using t.Setenv or t.Chdir can not use t.Parallel` + // Parallel signals that this test is to be run in parallel with (and only with) // other parallel tests. When a test is run multiple times due to use of // -test.count or -test.cpu, multiple instances of a single test never run in @@ -1444,8 +1537,8 @@ func (t *T) Parallel() { if t.isParallel { panic("testing: t.Parallel called multiple times") } - if t.isEnvSet { - panic("testing: t.Parallel called after t.Setenv; cannot set environment variables in parallel tests") + if t.denyParallel { + panic(parallelConflict) } t.isParallel = true if t.parent.barrier == nil { @@ -1482,7 +1575,7 @@ func (t *T) Parallel() { t.signal <- true // Release calling test. <-t.parent.barrier // Wait for the parent test to complete. - t.context.waitParallel() + t.tstate.waitParallel() if t.chatty != nil { t.chatty.Updatef(t.name, "=== CONT %s\n", t.name) @@ -1500,6 +1593,21 @@ func (t *T) Parallel() { t.lastRaceErrors.Store(int64(race.Errors())) } +func (t *T) checkParallel() { + // Non-parallel subtests that have parallel ancestors may still + // run in parallel with other tests: they are only non-parallel + // with respect to the other subtests of the same parent. + // Since calls like SetEnv or Chdir affects the whole process, we need + // to deny those if the current test or any parent is parallel. + for c := &t.common; c != nil; c = c.parent { + if c.isParallel { + panic(parallelConflict) + } + } + + t.denyParallel = true +} + // Setenv calls os.Setenv(key, value) and uses Cleanup to // restore the environment variable to its original value // after the test. @@ -1507,27 +1615,21 @@ func (t *T) Parallel() { // Because Setenv affects the whole process, it cannot be used // in parallel tests or tests with parallel ancestors. func (t *T) Setenv(key, value string) { - // Non-parallel subtests that have parallel ancestors may still - // run in parallel with other tests: they are only non-parallel - // with respect to the other subtests of the same parent. - // Since SetEnv affects the whole process, we need to disallow it - // if the current test or any parent is parallel. - isParallel := false - for c := &t.common; c != nil; c = c.parent { - if c.isParallel { - isParallel = true - break - } - } - if isParallel { - panic("testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests") - } - - t.isEnvSet = true - + t.checkParallel() t.common.Setenv(key, value) } +// Chdir calls os.Chdir(dir) and uses Cleanup to restore the current +// working directory to its original value after the test. On Unix, it +// also sets PWD environment variable for the duration of the test. +// +// Because Chdir affects the whole process, it cannot be used +// in parallel tests or tests with parallel ancestors. +func (t *T) Chdir(dir string) { + t.checkParallel() + t.common.Chdir(dir) +} + // InternalTest is an internal type but exported because it is cross-package; // it is part of the implementation of the "go test" command. type InternalTest struct { @@ -1583,7 +1685,7 @@ func tRunner(t *T, fn func(t *T)) { } } - if err != nil && t.context.isFuzzing { + if err != nil && t.tstate.isFuzzing { prefix := "panic: " if err == errNilPanicOrGoexit { prefix = "" @@ -1641,7 +1743,7 @@ func tRunner(t *T, fn func(t *T)) { // Run parallel subtests. // Decrease the running count for this test and mark it as no longer running. - t.context.release() + t.tstate.release() running.Delete(t.name) // Release the parallel subtests. @@ -1663,12 +1765,12 @@ func tRunner(t *T, fn func(t *T)) { t.checkRaces() if !t.isParallel { // Reacquire the count for sequential tests. See comment in Run. - t.context.waitParallel() + t.tstate.waitParallel() } } else if t.isParallel { // Only release the count for this test if it was run as a parallel // test. See comment in Run method. - t.context.release() + t.tstate.release() } t.report() // Report after all subtests have finished. @@ -1707,7 +1809,7 @@ func (t *T) Run(name string, f func(t *T)) bool { } t.hasSub.Store(true) - testName, ok, _ := t.context.match.fullName(&t.common, name) + testName, ok, _ := t.tstate.match.fullName(&t.common, name) if !ok || shouldFailFast() { return true } @@ -1716,17 +1818,23 @@ func (t *T) Run(name string, f func(t *T)) bool { // continue walking the stack into the parent test. var pc [maxStackLen]uintptr n := runtime.Callers(2, pc[:]) + + // There's no reason to inherit this context from parent. The user's code can't observe + // the difference between the background context and the one from the parent test. + ctx, cancelCtx := context.WithCancel(context.Background()) t = &T{ common: common{ - barrier: make(chan bool), - signal: make(chan bool, 1), - name: testName, - parent: &t.common, - level: t.level + 1, - creator: pc[:n], - chatty: t.chatty, + barrier: make(chan bool), + signal: make(chan bool, 1), + name: testName, + parent: &t.common, + level: t.level + 1, + creator: pc[:n], + chatty: t.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - context: t.context, + tstate: t.tstate, } t.w = indenter{&t.common} @@ -1765,17 +1873,17 @@ func (t *T) Run(name string, f func(t *T)) bool { // // The ok result is false if the -timeout flag indicates “no timeout” (0). func (t *T) Deadline() (deadline time.Time, ok bool) { - deadline = t.context.deadline + deadline = t.tstate.deadline return deadline, !deadline.IsZero() } -// testContext holds all fields that are common to all tests. This includes +// testState holds all fields that are common to all tests. This includes // synchronization primitives to run at most *parallel tests. -type testContext struct { +type testState struct { match *matcher deadline time.Time - // isFuzzing is true in the context used when generating random inputs + // isFuzzing is true in the state used when generating random inputs // for fuzz targets. isFuzzing is false when running normal tests and // when running fuzz tests as unit tests (without -fuzz or when -fuzz // does not match). @@ -1797,8 +1905,8 @@ type testContext struct { maxParallel int } -func newTestContext(maxParallel int, m *matcher) *testContext { - return &testContext{ +func newTestState(maxParallel int, m *matcher) *testState { + return &testState{ match: m, startParallel: make(chan bool), maxParallel: maxParallel, @@ -1806,28 +1914,28 @@ func newTestContext(maxParallel int, m *matcher) *testContext { } } -func (c *testContext) waitParallel() { - c.mu.Lock() - if c.running < c.maxParallel { - c.running++ - c.mu.Unlock() +func (s *testState) waitParallel() { + s.mu.Lock() + if s.running < s.maxParallel { + s.running++ + s.mu.Unlock() return } - c.numWaiting++ - c.mu.Unlock() - <-c.startParallel + s.numWaiting++ + s.mu.Unlock() + <-s.startParallel } -func (c *testContext) release() { - c.mu.Lock() - if c.numWaiting == 0 { - c.running-- - c.mu.Unlock() +func (s *testState) release() { + s.mu.Lock() + if s.numWaiting == 0 { + s.running-- + s.mu.Unlock() return } - c.numWaiting-- - c.mu.Unlock() - c.startParallel <- true // Pick a waiting test to be run. + s.numWaiting-- + s.mu.Unlock() + s.startParallel <- true // Pick a waiting test to be run. } // No one should be using func Main anymore. @@ -2150,15 +2258,18 @@ func runTests(matchString func(pat, str string) (bool, error), tests []InternalT // to keep trying. break } - ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run", *skip)) - ctx.deadline = deadline + ctx, cancelCtx := context.WithCancel(context.Background()) + tstate := newTestState(*parallel, newMatcher(matchString, *match, "-test.run", *skip)) + tstate.deadline = deadline t := &T{ common: common{ - signal: make(chan bool, 1), - barrier: make(chan bool), - w: os.Stdout, + signal: make(chan bool, 1), + barrier: make(chan bool), + w: os.Stdout, + ctx: ctx, + cancelCtx: cancelCtx, }, - context: ctx, + tstate: tstate, } if Verbose() { t.chatty = newChattyPrinter(t.w) diff --git a/src/testing/testing_test.go b/src/testing/testing_test.go index 4a930395..797728c7 100644 --- a/src/testing/testing_test.go +++ b/src/testing/testing_test.go @@ -6,6 +6,8 @@ package testing_test import ( "bytes" + "context" + "errors" "fmt" "internal/race" "internal/testenv" @@ -13,6 +15,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "slices" "strings" "sync" @@ -200,64 +203,177 @@ func TestSetenv(t *testing.T) { } } -func TestSetenvWithParallelAfterSetenv(t *testing.T) { - defer func() { - want := "testing: t.Parallel called after t.Setenv; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() +func expectParallelConflict(t *testing.T) { + want := testing.ParallelConflict + if got := recover(); got != want { + t.Fatalf("expected panic; got %#v want %q", got, want) + } +} - t.Setenv("GO_TEST_KEY_1", "value") +func testWithParallelAfter(t *testing.T, fn func(*testing.T)) { + defer expectParallelConflict(t) + fn(t) t.Parallel() } -func TestSetenvWithParallelBeforeSetenv(t *testing.T) { - defer func() { - want := "testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() +func testWithParallelBefore(t *testing.T, fn func(*testing.T)) { + defer expectParallelConflict(t) t.Parallel() - - t.Setenv("GO_TEST_KEY_1", "value") + fn(t) } -func TestSetenvWithParallelParentBeforeSetenv(t *testing.T) { +func testWithParallelParentBefore(t *testing.T, fn func(*testing.T)) { t.Parallel() t.Run("child", func(t *testing.T) { - defer func() { - want := "testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() + defer expectParallelConflict(t) - t.Setenv("GO_TEST_KEY_1", "value") + fn(t) }) } -func TestSetenvWithParallelGrandParentBeforeSetenv(t *testing.T) { +func testWithParallelGrandParentBefore(t *testing.T, fn func(*testing.T)) { t.Parallel() t.Run("child", func(t *testing.T) { t.Run("grand-child", func(t *testing.T) { - defer func() { - want := "testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() + defer expectParallelConflict(t) - t.Setenv("GO_TEST_KEY_1", "value") + fn(t) }) }) } +func tSetenv(t *testing.T) { + t.Setenv("GO_TEST_KEY_1", "value") +} + +func TestSetenvWithParallelAfter(t *testing.T) { + testWithParallelAfter(t, tSetenv) +} + +func TestSetenvWithParallelBefore(t *testing.T) { + testWithParallelBefore(t, tSetenv) +} + +func TestSetenvWithParallelParentBefore(t *testing.T) { + testWithParallelParentBefore(t, tSetenv) +} + +func TestSetenvWithParallelGrandParentBefore(t *testing.T) { + testWithParallelGrandParentBefore(t, tSetenv) +} + +func tChdir(t *testing.T) { + t.Chdir(t.TempDir()) +} + +func TestChdirWithParallelAfter(t *testing.T) { + testWithParallelAfter(t, tChdir) +} + +func TestChdirWithParallelBefore(t *testing.T) { + testWithParallelBefore(t, tChdir) +} + +func TestChdirWithParallelParentBefore(t *testing.T) { + testWithParallelParentBefore(t, tChdir) +} + +func TestChdirWithParallelGrandParentBefore(t *testing.T) { + testWithParallelGrandParentBefore(t, tChdir) +} + +func TestChdir(t *testing.T) { + oldDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + defer os.Chdir(oldDir) + + // The "relative" test case relies on tmp not being a symlink. + tmp, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + rel, err := filepath.Rel(oldDir, tmp) + if err != nil { + // If GOROOT is on C: volume and tmp is on the D: volume, there + // is no relative path between them, so skip that test case. + rel = "skip" + } + + for _, tc := range []struct { + name, dir, pwd string + extraChdir bool + }{ + { + name: "absolute", + dir: tmp, + pwd: tmp, + }, + { + name: "relative", + dir: rel, + pwd: tmp, + }, + { + name: "current (absolute)", + dir: oldDir, + pwd: oldDir, + }, + { + name: "current (relative) with extra os.Chdir", + dir: ".", + pwd: oldDir, + + extraChdir: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.dir == "skip" { + t.Skipf("skipping test because there is no relative path between %s and %s", oldDir, tmp) + } + if !filepath.IsAbs(tc.pwd) { + t.Fatalf("Bad tc.pwd: %q (must be absolute)", tc.pwd) + } + + t.Chdir(tc.dir) + + newDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if newDir != tc.pwd { + t.Fatalf("failed to chdir to %q: getwd: got %q, want %q", tc.dir, newDir, tc.pwd) + } + + switch runtime.GOOS { + case "windows", "plan9": + // Windows and Plan 9 do not use the PWD variable. + default: + if pwd := os.Getenv("PWD"); pwd != tc.pwd { + t.Fatalf("PWD: got %q, want %q", pwd, tc.pwd) + } + } + + if tc.extraChdir { + os.Chdir("..") + } + }) + + newDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if newDir != oldDir { + t.Fatalf("failed to restore wd to %s: getwd: %s", oldDir, newDir) + } + } +} + // testingTrueInInit is part of TestTesting. var testingTrueInInit = false @@ -324,12 +440,7 @@ func runTest(t *testing.T, test string) []byte { testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Skipf("can't find test executable: %v", err) - } - - cmd := testenv.Command(t, exe, "-test.run=^"+test+"$", "-test.bench="+test, "-test.v", "-test.parallel=2", "-test.benchtime=2x") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^"+test+"$", "-test.bench="+test, "-test.v", "-test.parallel=2", "-test.benchtime=2x") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") out, err := cmd.CombinedOutput() @@ -558,14 +669,7 @@ func TestRaceBeforeParallel(t *testing.T) { } func TestRaceBeforeTests(t *testing.T) { - testenv.MustHaveExec(t) - - exe, err := os.Executable() - if err != nil { - t.Skipf("can't find test executable: %v", err) - } - - cmd := testenv.Command(t, exe, "-test.run=^$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_RACE_BEFORE_TESTS=1") out, _ := cmd.CombinedOutput() @@ -596,6 +700,20 @@ func TestBenchmarkRace(t *testing.T) { } } +func TestBenchmarkRaceBLoop(t *testing.T) { + out := runTest(t, "BenchmarkBLoopRacy") + c := bytes.Count(out, []byte("race detected during execution of test")) + + want := 0 + // We should see one race detector report. + if race.Enabled { + want = 1 + } + if c != want { + t.Errorf("got %d race reports; want %d", c, want) + } +} + func BenchmarkRacy(b *testing.B) { if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { b.Skipf("skipping intentionally-racy benchmark") @@ -605,15 +723,25 @@ func BenchmarkRacy(b *testing.B) { } } +func BenchmarkBLoopRacy(b *testing.B) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + b.Skipf("skipping intentionally-racy benchmark") + } + for b.Loop() { + doRace() + } +} + func TestBenchmarkSubRace(t *testing.T) { out := runTest(t, "BenchmarkSubRacy") c := bytes.Count(out, []byte("race detected during execution of test")) want := 0 - // We should see two race detector reports: - // one in the sub-bencmark, and one in the parent afterward. + // We should see 3 race detector reports: + // one in the sub-bencmark, one in the parent afterward, + // and one in b.Loop. if race.Enabled { - want = 2 + want = 3 } if c != want { t.Errorf("got %d race reports; want %d", c, want) @@ -639,6 +767,12 @@ func BenchmarkSubRacy(b *testing.B) { } }) + b.Run("racy-bLoop", func(b *testing.B) { + for b.Loop() { + doRace() + } + }) + doRace() // should be reported separately } @@ -813,3 +947,85 @@ func TestParentRun(t1 *testing.T) { }) }) } + +func TestContext(t *testing.T) { + ctx := t.Context() + if err := ctx.Err(); err != nil { + t.Fatalf("expected non-canceled context, got %v", err) + } + + var innerCtx context.Context + t.Run("inner", func(t *testing.T) { + innerCtx = t.Context() + if err := innerCtx.Err(); err != nil { + t.Fatalf("expected inner test to not inherit canceled context, got %v", err) + } + }) + t.Run("inner2", func(t *testing.T) { + if !errors.Is(innerCtx.Err(), context.Canceled) { + t.Fatal("expected context of sibling test to be canceled after its test function finished") + } + }) + + t.Cleanup(func() { + if !errors.Is(ctx.Err(), context.Canceled) { + t.Fatal("expected context canceled before cleanup") + } + }) +} + +func TestBenchmarkBLoopIterationCorrect(t *testing.T) { + out := runTest(t, "BenchmarkBLoopPrint") + c := bytes.Count(out, []byte("Printing from BenchmarkBLoopPrint")) + + want := 2 + if c != want { + t.Errorf("got %d loop iterations; want %d", c, want) + } + + // b.Loop() will only rampup once. + c = bytes.Count(out, []byte("Ramping up from BenchmarkBLoopPrint")) + want = 1 + if c != want { + t.Errorf("got %d loop rampup; want %d", c, want) + } + + re := regexp.MustCompile(`BenchmarkBLoopPrint(-[0-9]+)?\s+2\s+[0-9]+\s+ns/op`) + if !re.Match(out) { + t.Error("missing benchmark output") + } +} + +func TestBenchmarkBNIterationCorrect(t *testing.T) { + out := runTest(t, "BenchmarkBNPrint") + c := bytes.Count(out, []byte("Printing from BenchmarkBNPrint")) + + // runTest sets benchtime=2x, with semantics specified in #32051 it should + // run 3 times. + want := 3 + if c != want { + t.Errorf("got %d loop iterations; want %d", c, want) + } + + // b.N style fixed iteration loop will rampup twice: + // One in run1(), the other in launch + c = bytes.Count(out, []byte("Ramping up from BenchmarkBNPrint")) + want = 2 + if c != want { + t.Errorf("got %d loop rampup; want %d", c, want) + } +} + +func BenchmarkBLoopPrint(b *testing.B) { + b.Logf("Ramping up from BenchmarkBLoopPrint") + for b.Loop() { + b.Logf("Printing from BenchmarkBLoopPrint") + } +} + +func BenchmarkBNPrint(b *testing.B) { + b.Logf("Ramping up from BenchmarkBNPrint") + for i := 0; i < b.N; i++ { + b.Logf("Printing from BenchmarkBNPrint") + } +} diff --git a/src/text/template/doc.go b/src/text/template/doc.go index b3ffaabb..7b63bb76 100644 --- a/src/text/template/doc.go +++ b/src/text/template/doc.go @@ -98,7 +98,8 @@ data, defined in detail in the corresponding sections that follow. {{if pipeline}} T1 {{else}}{{if pipeline}} T0 {{end}}{{end}} {{range pipeline}} T1 {{end}} - The value of the pipeline must be an array, slice, map, or channel. + The value of the pipeline must be an array, slice, map, iter.Seq, + iter.Seq2, integer or channel. If the value of the pipeline has length zero, nothing is output; otherwise, dot is set to the successive elements of the array, slice, or map and T1 is executed. If the value is a map and the @@ -106,7 +107,8 @@ data, defined in detail in the corresponding sections that follow. visited in sorted key order. {{range pipeline}} T1 {{else}} T0 {{end}} - The value of the pipeline must be an array, slice, map, or channel. + The value of the pipeline must be an array, slice, map, iter.Seq, + iter.Seq2, integer or channel. If the value of the pipeline has length zero, dot is unaffected and T0 is executed; otherwise, dot is set to the successive elements of the array, slice, or map and T1 is executed. @@ -162,37 +164,55 @@ An argument is a simple value, denoted by one of the following. the host machine's ints are 32 or 64 bits. - The keyword nil, representing an untyped Go nil. - The character '.' (period): + . + The result is the value of dot. - A variable name, which is a (possibly empty) alphanumeric string preceded by a dollar sign, such as + $piOver2 + or + $ + The result is the value of the variable. Variables are described below. - The name of a field of the data, which must be a struct, preceded by a period, such as + .Field + The result is the value of the field. Field invocations may be chained: + .Field1.Field2 + Fields can also be evaluated on variables, including chaining: + $x.Field1.Field2 - The name of a key of the data, which must be a map, preceded by a period, such as + .Key + The result is the map element value indexed by the key. Key invocations may be chained and combined with fields to any depth: + .Field1.Key1.Field2.Key2 + Although the key must be an alphanumeric identifier, unlike with field names they do not need to start with an upper case letter. Keys can also be evaluated on variables, including chaining: + $x.key1.key2 - The name of a niladic method of the data, preceded by a period, such as + .Method + The result is the value of invoking the method with dot as the receiver, dot.Method(). Such a method must have one return value (of any type) or two return values, the second of which is an error. @@ -200,16 +220,22 @@ An argument is a simple value, denoted by one of the following. and an error is returned to the caller as the value of Execute. Method invocations may be chained and combined with fields and keys to any depth: + .Field1.Key1.Method1.Field2.Key2.Method2 + Methods can also be evaluated on variables, including chaining: + $x.Method1.Field - The name of a niladic function, such as + fun + The result is the value of invoking the function, fun(). The return types and values behave as in methods. Functions and function names are described below. - A parenthesized instance of one the above, for grouping. The result may be accessed by a field or map key invocation. + print (.F1 arg1) (.F2 arg2) (.StructValuedMethod "arg").Field diff --git a/src/text/template/exec.go b/src/text/template/exec.go index 5b35b3e5..ed6ae436 100644 --- a/src/text/template/exec.go +++ b/src/text/template/exec.go @@ -395,6 +395,22 @@ func (s *state) walkRange(dot reflect.Value, r *parse.RangeNode) { s.walk(elem, r.List) } switch val.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + if len(r.Pipe.Decl) > 1 { + s.errorf("can't use %v to iterate over more than one variable", val) + break + } + run := false + for v := range val.Seq() { + run = true + // Pass element as second value, as we do for channels. + oneIteration(reflect.Value{}, v) + } + if !run { + break + } + return case reflect.Array, reflect.Slice: if val.Len() == 0 { break @@ -434,6 +450,43 @@ func (s *state) walkRange(dot reflect.Value, r *parse.RangeNode) { return case reflect.Invalid: break // An invalid value is likely a nil map, etc. and acts like an empty map. + case reflect.Func: + if val.Type().CanSeq() { + if len(r.Pipe.Decl) > 1 { + s.errorf("can't use %v iterate over more than one variable", val) + break + } + run := false + for v := range val.Seq() { + run = true + // Pass element as second value, + // as we do for channels. + oneIteration(reflect.Value{}, v) + } + if !run { + break + } + return + } + if val.Type().CanSeq2() { + run := false + for i, v := range val.Seq2() { + run = true + if len(r.Pipe.Decl) > 1 { + oneIteration(i, v) + } else { + // If there is only one range variable, + // oneIteration will use the + // second value. + oneIteration(reflect.Value{}, i) + } + } + if !run { + break + } + return + } + fallthrough default: s.errorf("range can't iterate over %v", val) } @@ -757,7 +810,7 @@ func (s *state) evalCall(dot, fun reflect.Value, isBuiltin bool, node parse.Node return v } } - if final != missingVal { + if !final.Equal(missingVal) { // The last argument to and/or is coming from // the pipeline. We didn't short circuit on an earlier // argument, so we are going to return this one. @@ -803,7 +856,13 @@ func (s *state) evalCall(dot, fun reflect.Value, isBuiltin bool, node parse.Node // Special case for the "call" builtin. // Insert the name of the callee function as the first argument. if isBuiltin && name == "call" { - calleeName := args[0].String() + var calleeName string + if len(args) == 0 { + // final must be present or we would have errored out above. + calleeName = final.String() + } else { + calleeName = args[0].String() + } argv = append([]reflect.Value{reflect.ValueOf(calleeName)}, argv...) fun = reflect.ValueOf(call) } diff --git a/src/text/template/exec_test.go b/src/text/template/exec_test.go index 9903e17d..03ec9d75 100644 --- a/src/text/template/exec_test.go +++ b/src/text/template/exec_test.go @@ -10,6 +10,7 @@ import ( "flag" "fmt" "io" + "iter" "reflect" "strings" "sync" @@ -399,6 +400,9 @@ var execTests = []execTest{ {"Interface Call", `{{stringer .S}}`, "foozle", map[string]any{"S": bytes.NewBufferString("foozle")}, true}, {".ErrFunc", "{{call .ErrFunc}}", "bla", tVal, true}, {"call nil", "{{call nil}}", "", tVal, false}, + {"empty call", "{{call}}", "", tVal, false}, + {"empty call after pipe valid", "{{.ErrFunc | call}}", "bla", tVal, true}, + {"empty call after pipe invalid", "{{1 | call}}", "", tVal, false}, // Erroneous function calls (check args). {".BinaryFuncTooFew", "{{call .BinaryFunc `1`}}", "", tVal, false}, @@ -601,6 +605,30 @@ var execTests = []execTest{ {"declare in range", "{{range $x := .PSI}}<{{$foo:=$x}}{{$x}}>{{end}}", "<21><22><23>", tVal, true}, {"range count", `{{range $i, $x := count 5}}[{{$i}}]{{$x}}{{end}}`, "[0]a[1]b[2]c[3]d[4]e", tVal, true}, {"range nil count", `{{range $i, $x := count 0}}{{else}}empty{{end}}`, "empty", tVal, true}, + {"range iter.Seq[int]", `{{range $i := .}}{{$i}}{{end}}`, "01", fVal1(2), true}, + {"i = range iter.Seq[int]", `{{$i := 0}}{{range $i = .}}{{$i}}{{end}}`, "01", fVal1(2), true}, + {"range iter.Seq[int] over two var", `{{range $i, $c := .}}{{$c}}{{end}}`, "", fVal1(2), false}, + {"i, c := range iter.Seq2[int,int]", `{{range $i, $c := .}}{{$i}}{{$c}}{{end}}`, "0112", fVal2(2), true}, + {"i, c = range iter.Seq2[int,int]", `{{$i := 0}}{{$c := 0}}{{range $i, $c = .}}{{$i}}{{$c}}{{end}}`, "0112", fVal2(2), true}, + {"i = range iter.Seq2[int,int]", `{{$i := 0}}{{range $i = .}}{{$i}}{{end}}`, "01", fVal2(2), true}, + {"i := range iter.Seq2[int,int]", `{{range $i := .}}{{$i}}{{end}}`, "01", fVal2(2), true}, + {"i,c,x range iter.Seq2[int,int]", `{{$i := 0}}{{$c := 0}}{{$x := 0}}{{range $i, $c = .}}{{$i}}{{$c}}{{end}}`, "0112", fVal2(2), true}, + {"i,x range iter.Seq[int]", `{{$i := 0}}{{$x := 0}}{{range $i = .}}{{$i}}{{end}}`, "01", fVal1(2), true}, + {"range iter.Seq[int] else", `{{range $i := .}}{{$i}}{{else}}empty{{end}}`, "empty", fVal1(0), true}, + {"range iter.Seq2[int,int] else", `{{range $i := .}}{{$i}}{{else}}empty{{end}}`, "empty", fVal2(0), true}, + {"range int8", rangeTestInt, rangeTestData[int8](), int8(5), true}, + {"range int16", rangeTestInt, rangeTestData[int16](), int16(5), true}, + {"range int32", rangeTestInt, rangeTestData[int32](), int32(5), true}, + {"range int64", rangeTestInt, rangeTestData[int64](), int64(5), true}, + {"range int", rangeTestInt, rangeTestData[int](), int(5), true}, + {"range uint8", rangeTestInt, rangeTestData[uint8](), uint8(5), true}, + {"range uint16", rangeTestInt, rangeTestData[uint16](), uint16(5), true}, + {"range uint32", rangeTestInt, rangeTestData[uint32](), uint32(5), true}, + {"range uint64", rangeTestInt, rangeTestData[uint64](), uint64(5), true}, + {"range uint", rangeTestInt, rangeTestData[uint](), uint(5), true}, + {"range uintptr", rangeTestInt, rangeTestData[uintptr](), uintptr(5), true}, + {"range uintptr(0)", `{{range $v := .}}{{print $v}}{{else}}empty{{end}}`, "empty", uintptr(0), true}, + {"range 5", `{{range $v := 5}}{{printf "%T%d" $v $v}}{{end}}`, rangeTestData[int](), nil, true}, // Cute examples. {"or as if true", `{{or .SI "slice is empty"}}`, "[3 4 5]", tVal, true}, @@ -705,6 +733,37 @@ var execTests = []execTest{ {"issue60801", "{{$k := 0}}{{$v := 0}}{{range $k, $v = .AI}}{{$k}}={{$v}} {{end}}", "0=3 1=4 2=5 ", tVal, true}, } +func fVal1(i int) iter.Seq[int] { + return func(yield func(int) bool) { + for v := range i { + if !yield(v) { + break + } + } + } +} + +func fVal2(i int) iter.Seq2[int, int] { + return func(yield func(int, int) bool) { + for v := range i { + if !yield(v, v+1) { + break + } + } + } +} + +const rangeTestInt = `{{range $v := .}}{{printf "%T%d" $v $v}}{{end}}` + +func rangeTestData[T int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | uintptr]() string { + I := T(5) + var buf strings.Builder + for i := T(0); i < I; i++ { + fmt.Fprintf(&buf, "%T%d", i, i) + } + return buf.String() +} + func zeroArgs() string { return "zeroArgs" } diff --git a/src/text/template/parse/lex.go b/src/text/template/parse/lex.go index 70fc86b6..a00f48e6 100644 --- a/src/text/template/parse/lex.go +++ b/src/text/template/parse/lex.go @@ -352,6 +352,7 @@ func lexComment(l *lexer) stateFn { if !delim { return l.errorf("comment ends before closing delimiter") } + l.line += strings.Count(l.input[l.start:l.pos], "\n") i := l.thisItem(itemComment) if trimSpace { l.pos += trimMarkerLen diff --git a/src/text/template/parse/lex_test.go b/src/text/template/parse/lex_test.go index d47f10f9..20f9698f 100644 --- a/src/text/template/parse/lex_test.go +++ b/src/text/template/parse/lex_test.go @@ -545,6 +545,16 @@ var lexPosTests = []lexTest{ {itemRightDelim, 11, "}}", 2}, {itemEOF, 13, "", 2}, }}, + {"longcomment", "{{/*\n*/}}\n{{undefinedFunction \"test\"}}", []item{ + {itemComment, 2, "/*\n*/", 1}, + {itemText, 9, "\n", 2}, + {itemLeftDelim, 10, "{{", 3}, + {itemIdentifier, 12, "undefinedFunction", 3}, + {itemSpace, 29, " ", 3}, + {itemString, 30, "\"test\"", 3}, + {itemRightDelim, 36, "}}", 3}, + {itemEOF, 38, "", 3}, + }}, } // The other tests don't check position, to make the test cases easier to construct. diff --git a/src/text/template/template.go b/src/text/template/template.go index 86fd3f12..78067af2 100644 --- a/src/text/template/template.go +++ b/src/text/template/template.go @@ -5,6 +5,7 @@ package template import ( + "maps" "reflect" "sync" "text/template/parse" @@ -102,12 +103,8 @@ func (t *Template) Clone() (*Template, error) { } t.muFuncs.RLock() defer t.muFuncs.RUnlock() - for k, v := range t.parseFuncs { - nt.parseFuncs[k] = v - } - for k, v := range t.execFuncs { - nt.execFuncs[k] = v - } + maps.Copy(nt.parseFuncs, t.parseFuncs) + maps.Copy(nt.execFuncs, t.execFuncs) return nt, nil } diff --git a/src/time/abs_test.go b/src/time/abs_test.go new file mode 100644 index 00000000..61d093a3 --- /dev/null +++ b/src/time/abs_test.go @@ -0,0 +1,183 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time + +type testingT interface { + Error(args ...any) + Errorf(format string, args ...any) + Fail() + FailNow() + Failed() bool + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Log(args ...any) + Logf(format string, args ...any) + Skip(args ...any) + SkipNow() + Skipf(format string, args ...any) +} + +var InternalTests = []struct { + Name string + Test func(testingT) +}{ + {"AbsDaysSplit", testAbsDaysSplit}, + {"AbsYdaySplit", testAbsYdaySplit}, + {"AbsDate", testAbsDate}, + {"DateToAbsDays", testDateToAbsDays}, + {"DaysIn", testDaysIn}, + {"DaysBefore", testDaysBefore}, +} + +func testAbsDaysSplit(t testingT) { + isLeap := func(year uint64) bool { + return year%4 == 0 && (year%100 != 0 || year%400 == 0) + } + bad := 0 + wantYear := uint64(0) + wantYday := absYday(0) + for days := range absDays(1e6) { + century, cyear, yday := days.split() + if century != absCentury(wantYear/100) || cyear != absCyear(wantYear%100) || yday != wantYday { + t.Errorf("absDays(%d).split() = %d, %d, %d, want %d, %d, %d", + days, century, cyear, yday, + wantYear/100, wantYear%100, wantYday) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + end := absYday(365) + if isLeap(wantYear + 1) { + end = 366 + } + if wantYday++; wantYday == end { + wantYear++ + wantYday = 0 + } + } +} + +func testAbsYdaySplit(t testingT) { + ends := []int{31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31, 29} + bad := 0 + wantMonth := absMonth(3) + wantDay := 1 + for yday := range absYday(366) { + month, day := yday.split() + if month != wantMonth || day != wantDay { + t.Errorf("absYday(%d).split() = %d, %d, want %d, %d", yday, month, day, wantMonth, wantDay) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + if wantDay++; wantDay > ends[wantMonth-3] { + wantMonth++ + wantDay = 1 + } + } +} + +func testAbsDate(t testingT) { + ends := []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} + isLeap := func(year int) bool { + y := uint64(year) + absoluteYears + return y%4 == 0 && (y%100 != 0 || y%400 == 0) + } + wantYear := 0 + wantMonth := March + wantMday := 1 + wantYday := 31 + 29 + 1 + bad := 0 + absoluteYears := int64(absoluteYears) + for days := range absDays(1e6) { + year, month, mday := days.date() + year += int(absoluteYears) + if year != wantYear || month != wantMonth || mday != wantMday { + t.Errorf("days(%d).date() = %v, %v, %v, want %v, %v, %v", days, + year, month, mday, + wantYear, wantMonth, wantMday) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + + year, yday := days.yearYday() + year += int(absoluteYears) + if year != wantYear || yday != wantYday { + t.Errorf("days(%d).yearYday() = %v, %v, want %v, %v, ", days, + year, yday, + wantYear, wantYday) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + + if wantMday++; wantMday == ends[wantMonth-1]+1 || wantMonth == February && wantMday == 29 && !isLeap(year) { + wantMonth++ + wantMday = 1 + } + wantYday++ + if wantMonth == December+1 { + wantYear++ + wantMonth = January + wantMday = 1 + wantYday = 1 + } + } +} + +func testDateToAbsDays(t testingT) { + isLeap := func(year int64) bool { + return year%4 == 0 && (year%100 != 0 || year%400 == 0) + } + wantDays := absDays(marchThruDecember) + bad := 0 + for year := int64(1); year < 10000; year++ { + days := dateToAbsDays(year-absoluteYears, January, 1) + if days != wantDays { + t.Errorf("dateToAbsDays(abs %d, Jan, 1) = %d, want %d", year, days, wantDays) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + wantDays += 365 + if isLeap(year) { + wantDays++ + } + } +} + +func testDaysIn(t testingT) { + isLeap := func(year int) bool { + return year%4 == 0 && (year%100 != 0 || year%400 == 0) + } + want := []int{0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} + bad := 0 + for year := 0; year <= 1600; year++ { + for m := January; m <= December; m++ { + w := want[m] + if m == February && isLeap(year) { + w++ + } + d := daysIn(m, year-800) + if d != w { + t.Errorf("daysIn(%v, %d) = %d, want %d", m, year-800, d, w) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + } + } +} + +func testDaysBefore(t testingT) { + for m, want := range []int{0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365} { + d := daysBefore(Month(m + 1)) + if d != want { + t.Errorf("daysBefore(%d) = %d, want %d", m, d, want) + } + } +} diff --git a/src/time/example_test.go b/src/time/example_test.go index 53c20a05..2c9601c6 100644 --- a/src/time/example_test.go +++ b/src/time/example_test.go @@ -6,6 +6,7 @@ package time_test import ( "fmt" + "math" "time" ) @@ -108,6 +109,39 @@ func ExampleParseDuration() { // There are 1.00e-06 seconds in 1µs. } +func ExampleSince() { + start := time.Now() + expensiveCall() + elapsed := time.Since(start) + fmt.Printf("The call took %v to run.\n", elapsed) +} + +func ExampleUntil() { + futureTime := time.Now().Add(5 * time.Second) + durationUntil := time.Until(futureTime) + fmt.Printf("Duration until future time: %.0f seconds", math.Ceil(durationUntil.Seconds())) + // Output: Duration until future time: 5 seconds +} + +func ExampleDuration_Abs() { + positiveDuration := 5 * time.Second + negativeDuration := -3 * time.Second + minInt64CaseDuration := time.Duration(math.MinInt64) + + absPositive := positiveDuration.Abs() + absNegative := negativeDuration.Abs() + absSpecial := minInt64CaseDuration.Abs() == time.Duration(math.MaxInt64) + + fmt.Printf("Absolute value of positive duration: %v\n", absPositive) + fmt.Printf("Absolute value of negative duration: %v\n", absNegative) + fmt.Printf("Absolute value of MinInt64 equal to MaxInt64: %t\n", absSpecial) + + // Output: + // Absolute value of positive duration: 5s + // Absolute value of negative duration: 3s + // Absolute value of MinInt64 equal to MaxInt64: true +} + func ExampleDuration_Hours() { h, _ := time.ParseDuration("4h30m") fmt.Printf("I've got %.1f hours of work left.", h.Hours()) @@ -295,8 +329,8 @@ func ExampleTime_Format() { // default format: 2015-02-25 11:06:39 -0800 PST // Unix format: Wed Feb 25 11:06:39 PST 2015 // Same, in UTC: Wed Feb 25 19:06:39 UTC 2015 - //in Shanghai with seconds: 2015-02-26T03:06:39 +080000 - //in Shanghai with colon seconds: 2015-02-26T03:06:39 +08:00:00 + // in Shanghai with seconds: 2015-02-26T03:06:39 +080000 + // in Shanghai with colon seconds: 2015-02-26T03:06:39 +08:00:00 // // Formats: // diff --git a/src/time/format.go b/src/time/format.go index c9e68b3e..da1bac5a 100644 --- a/src/time/format.go +++ b/src/time/format.go @@ -140,7 +140,7 @@ const ( stdDay // "2" stdUnderDay // "_2" stdZeroDay // "02" - stdUnderYearDay // "__2" + stdUnderYearDay = iota + stdNeedYday // "__2" stdZeroYearDay // "002" stdHour = iota + stdNeedClock // "15" stdHour12 // "3" @@ -168,7 +168,8 @@ const ( stdFracSecond9 // ".9", ".99", ..., trailing zeros omitted stdNeedDate = 1 << 8 // need month, day, year - stdNeedClock = 2 << 8 // need hour, minute, second + stdNeedYday = 1 << 9 // need yday + stdNeedClock = 1 << 10 // need hour, minute, second stdArgShift = 16 // extra argument in high bits, above low stdArgShift stdSeparatorShift = 28 // extra argument in high 4 bits for fractional second separators stdMask = 1<= i+2 && layout[i+1] == '2' { - //_2006 is really a literal _, followed by stdLongYear + // _2006 is really a literal _, followed by stdLongYear if len(layout) >= i+5 && layout[i+1:i+5] == "2006" { return layout[0 : i+1], stdLongYear, layout[i+5:] } @@ -404,7 +405,7 @@ func match(s1, s2 string) bool { func lookup(tab []string, val string) (int, string, error) { for i, v := range tab { - if len(val) >= len(v) && match(val[0:len(v)], v) { + if len(val) >= len(v) && match(val[:len(v)], v) { return i, val[len(v):], nil } } @@ -574,9 +575,9 @@ func (t Time) String() string { // GoString implements [fmt.GoStringer] and formats t to be printed in Go source // code. func (t Time) GoString() string { - abs := t.abs() - year, month, day, _ := absDate(abs, true) - hour, minute, second := absClock(abs) + abs := t.absSec() + year, month, day := abs.days().date() + hour, minute, second := abs.clock() buf := make([]byte, 0, len("time.Date(9999, time.September, 31, 23, 59, 59, 999999999, time.Local)")) buf = append(buf, "time.Date("...) @@ -664,13 +665,14 @@ func (t Time) AppendFormat(b []byte, layout string) []byte { } func (t Time) appendFormat(b []byte, layout string) []byte { - var ( - name, offset, abs = t.locabs() + name, offset, abs := t.locabs() + days := abs.days() + var ( year int = -1 month Month day int - yday int + yday int = -1 hour int = -1 min int sec int @@ -689,13 +691,15 @@ func (t Time) appendFormat(b []byte, layout string) []byte { // Compute year, month, day if needed. if year < 0 && std&stdNeedDate != 0 { - year, month, day, yday = absDate(abs, true) - yday++ + year, month, day = days.date() + } + if yday < 0 && std&stdNeedYday != 0 { + _, yday = days.yearYday() } // Compute hour, minute, second if needed. if hour < 0 && std&stdNeedClock != 0 { - hour, min, sec = absClock(abs) + hour, min, sec = abs.clock() } switch std & stdMask { @@ -717,9 +721,9 @@ func (t Time) appendFormat(b []byte, layout string) []byte { case stdZeroMonth: b = appendInt(b, int(month), 2) case stdWeekDay: - b = append(b, absWeekday(abs).String()[:3]...) + b = append(b, days.weekday().String()[:3]...) case stdLongWeekDay: - s := absWeekday(abs).String() + s := days.weekday().String() b = append(b, s...) case stdDay: b = appendInt(b, day, 0) @@ -1255,9 +1259,9 @@ func parse(layout, value string, defaultLocation, local *Location) (Time, error) hr, _, err = getnum(hour, true) if err == nil { mm, _, err = getnum(min, true) - } - if err == nil { - ss, _, err = getnum(seconds, true) + if err == nil { + ss, _, err = getnum(seconds, true) + } } // The range test use > rather than >=, @@ -1350,10 +1354,10 @@ func parse(layout, value string, defaultLocation, local *Location) (Time, error) } if m == 0 { m = (yday-1)/31 + 1 - if int(daysBefore[m]) < yday { + if daysBefore(Month(m+1)) < yday { m++ } - d = yday - int(daysBefore[m-1]) + d = yday - daysBefore(Month(m)) } // If month, day already seen, yday's m, d must match. // Otherwise, set them from m, d. diff --git a/src/time/format_rfc3339.go b/src/time/format_rfc3339.go index 1151666c..05fddfca 100644 --- a/src/time/format_rfc3339.go +++ b/src/time/format_rfc3339.go @@ -19,7 +19,7 @@ func (t Time) appendFormatRFC3339(b []byte, nanos bool) []byte { _, offset, abs := t.locabs() // Format date. - year, month, day, _ := absDate(abs, true) + year, month, day := abs.days().date() b = appendInt(b, year, 4) b = append(b, '-') b = appendInt(b, int(month), 2) @@ -29,7 +29,7 @@ func (t Time) appendFormatRFC3339(b []byte, nanos bool) []byte { b = append(b, 'T') // Format time. - hour, min, sec := absClock(abs) + hour, min, sec := abs.clock() b = appendInt(b, hour, 2) b = append(b, ':') b = appendInt(b, min, 2) diff --git a/src/time/linkname_test.go b/src/time/linkname_test.go new file mode 100644 index 00000000..3a47eaf3 --- /dev/null +++ b/src/time/linkname_test.go @@ -0,0 +1,41 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time_test + +import ( + "testing" + "time" + _ "unsafe" // for linkname +) + +//go:linkname timeAbs time.Time.abs +func timeAbs(time.Time) uint64 + +//go:linkname absClock time.absClock +func absClock(uint64) (hour, min, sec int) + +//go:linkname absDate time.absDate +func absDate(uint64, bool) (year int, month time.Month, day int, yday int) + +func TestLinkname(t *testing.T) { + tm := time.Date(2006, time.January, 2, 15, 4, 5, 6, time.UTC) + abs := timeAbs(tm) + // wantAbs should be Jan 1 based, not Mar 1 based. + // See absolute time description in time.go. + const wantAbs = 9223372029851535845 // NOT 9223372029877973939 + if abs != wantAbs { + t.Fatalf("timeAbs(2006-01-02 15:04:05 UTC) = %d, want %d", abs, uint64(wantAbs)) + } + + year, month, day, yday := absDate(abs, true) + if year != 2006 || month != time.January || day != 2 || yday != 1 { + t.Errorf("absDate() = %v, %v, %v, %v, want 2006, January, 2, 1", year, month, day, yday) + } + + hour, min, sec := absClock(abs) + if hour != 15 || min != 4 || sec != 5 { + t.Errorf("absClock() = %v, %v, %v, 15, 4, 5", hour, min, sec) + } +} diff --git a/src/time/sleep.go b/src/time/sleep.go index 7e2fa0c2..e9cd483b 100644 --- a/src/time/sleep.go +++ b/src/time/sleep.go @@ -165,8 +165,8 @@ func NewTimer(d Duration) *Timer { // to receive a time value corresponding to the previous timer settings; // if the program has not received from t.C already and the timer is // running, Reset is guaranteed to return true. -// Before Go 1.23, the only safe way to use Reset was to [Stop] and -// explicitly drain the timer first. +// Before Go 1.23, the only safe way to use Reset was to call [Timer.Stop] +// and explicitly drain the timer first. // See the [NewTimer] documentation for more details. func (t *Timer) Reset(d Duration) bool { if !t.initTimer { @@ -180,7 +180,7 @@ func (t *Timer) Reset(d Duration) bool { func sendTime(c any, seq uintptr, delta int64) { // delta is how long ago the channel send was supposed to happen. // The current time can be arbitrarily far into the future, because the runtime - // can delay a sendTime call until a goroutines tries to receive from + // can delay a sendTime call until a goroutine tries to receive from // the channel. Subtract delta to go back to the old time that we // used to send. select { diff --git a/src/time/tick_test.go b/src/time/tick_test.go index fce9002c..416bef59 100644 --- a/src/time/tick_test.go +++ b/src/time/tick_test.go @@ -462,7 +462,7 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { tim.Reset(1) Sleep(sched) if l, c := len(C), cap(C); l != 0 || c != 0 { - //t.Fatalf("len(C), cap(C) = %d, %d, want 0, 0", l, c) + // t.Fatalf("len(C), cap(C) = %d, %d, want 0, 0", l, c) } assertTick() } else { diff --git a/src/time/time.go b/src/time/time.go index 43efe4b1..14e79672 100644 --- a/src/time/time.go +++ b/src/time/time.go @@ -54,8 +54,8 @@ // On some systems the monotonic clock will stop if the computer goes to sleep. // On such a system, t.Sub(u) may not accurately reflect the actual // time that passed between t and u. The same applies to other functions and -// methods that subtract times, such as [Since], [Until], [Before], [After], -// [Add], [Sub], [Equal] and [Compare]. In some cases, you may need to strip +// methods that subtract times, such as [Since], [Until], [Time.Before], [Time.After], +// [Time.Add], [Time.Equal] and [Time.Compare]. In some cases, you may need to strip // the monotonic clock to get accurate results. // // Because the monotonic clock reading has no meaning outside @@ -92,6 +92,7 @@ package time import ( "errors" + "math/bits" _ "unsafe" // for go:linkname ) @@ -118,9 +119,9 @@ import ( // these methods does not change the actual instant it represents, only the time // zone in which to interpret it. // -// Representations of a Time value saved by the [Time.GobEncode], [Time.MarshalBinary], -// [Time.MarshalJSON], and [Time.MarshalText] methods store the [Time.Location]'s offset, but not -// the location name. They therefore lose information about Daylight Saving Time. +// Representations of a Time value saved by the [Time.GobEncode], [Time.MarshalBinary], [Time.AppendBinary], +// [Time.MarshalJSON], [Time.MarshalText] and [Time.AppendText] methods store the [Time.Location]'s offset, +// but not the location name. They therefore lose information about Daylight Saving Time. // // In addition to the required “wall clock” reading, a Time may contain an optional // reading of the current process's monotonic clock, to provide additional precision @@ -256,6 +257,12 @@ func (t *Time) mono() int64 { return t.ext } +// IsZero reports whether t represents the zero time instant, +// January 1, year 1, 00:00:00 UTC. +func (t Time) IsZero() bool { + return t.sec() == 0 && t.nsec() == 0 +} + // After reports whether the time instant t is after u. func (t Time) After(u Time) bool { if t.wall&u.wall&hasMonotonic != 0 { @@ -360,7 +367,7 @@ func (d Weekday) String() string { return "%!Weekday(" + string(buf[n:]) + ")" } -// Computations on time. +// Computations on Times // // The zero value for a Time is defined to be // January 1, year 1, 00:00:00.000000000 UTC @@ -406,22 +413,22 @@ func (d Weekday) String() string { // // The calendar runs on an exact 400 year cycle: a 400-year calendar // printed for 1970-2369 will apply as well to 2370-2769. Even the days -// of the week match up. It simplifies the computations to choose the +// of the week match up. It simplifies date computations to choose the // cycle boundaries so that the exceptional years are always delayed as -// long as possible. That means choosing a year equal to 1 mod 400, so -// that the first leap year is the 4th year, the first missed leap year -// is the 100th year, and the missed missed leap year is the 400th year. -// So we'd prefer instead to print a calendar for 2001-2400 and reuse it -// for 2401-2800. +// long as possible: March 1, year 0 is such a day: +// the first leap day (Feb 29) is four years minus one day away, +// the first multiple-of-4 year without a Feb 29 is 100 years minus one day away, +// and the first multiple-of-100 year with a Feb 29 is 400 years minus one day away. +// March 1 year Y for any Y = 0 mod 400 is also such a day. // // Finally, it's convenient if the delta between the Unix epoch and // long-ago epoch is representable by an int64 constant. // // These three considerations—choose an epoch as early as possible, that -// uses a year equal to 1 mod 400, and that is no more than 2⁶³ seconds -// earlier than 1970—bring us to the year -292277022399. We refer to -// this year as the absolute zero year, and to times measured as a uint64 -// seconds since this year as absolute times. +// starts on March 1 of a year equal to 0 mod 400, and that is no more than +// 2⁶³ seconds earlier than 1970—bring us to the year -292277022400. +// We refer to this moment as the absolute zero instant, and to times +// measured as a uint64 seconds since this year as absolute times. // // Times measured as an int64 seconds since the year 1—the representation // used for Time's sec field—are called internal times. @@ -435,40 +442,327 @@ func (d Weekday) String() string { // west of UTC, since it is year 0. It doesn't seem tenable to say that // printing the zero time correctly isn't supported in half the time // zones. By comparison, it's reasonable to mishandle some times in -// the year -292277022399. +// the year -292277022400. // // All this is opaque to clients of the API and can be changed if a // better implementation presents itself. +// +// The date calculations are implemented using the following clever math from +// Cassio Neri and Lorenz Schneider, “Euclidean affine functions and their +// application to calendar algorithms,” SP&E 2023. https://doi.org/10.1002/spe.3172 +// +// Define a “calendrical division” (f, f°, f*) to be a triple of functions converting +// one time unit into a whole number of larger units and the remainder and back. +// For example, in a calendar with no leap years, (d/365, d%365, y*365) is the +// calendrical division for days into years: +// +// (f) year := days/365 +// (f°) yday := days%365 +// (f*) days := year*365 (+ yday) +// +// Note that f* is usually the “easy” function to write: it's the +// calendrical multiplication that inverts the more complex division. +// +// Neri and Schneider prove that when f* takes the form +// +// f*(n) = (a n + b) / c +// +// using integer division rounding down with a ≥ c > 0, +// which they call a Euclidean affine function or EAF, then: +// +// f(n) = (c n + c - b - 1) / a +// f°(n) = (c n + c - b - 1) % a / c +// +// This gives a fairly direct calculation for any calendrical division for which +// we can write the calendrical multiplication in EAF form. +// Because the epoch has been shifted to March 1, all the calendrical +// multiplications turn out to be possible to write in EAF form. +// When a date is broken into [century, cyear, amonth, mday], +// with century, cyear, and mday 0-based, +// and amonth 3-based (March = 3, ..., January = 13, February = 14), +// the calendrical multiplications written in EAF form are: +// +// yday = (153 (amonth-3) + 2) / 5 = (153 amonth - 457) / 5 +// cday = 365 cyear + cyear/4 = 1461 cyear / 4 +// centurydays = 36524 century + century/4 = 146097 century / 4 +// days = centurydays + cday + yday + mday. +// +// We can only handle one periodic cycle per equation, so the year +// calculation must be split into [century, cyear], handling both the +// 100-year cycle and the 400-year cycle. +// +// The yday calculation is not obvious but derives from the fact +// that the March through January calendar repeats the 5-month +// 153-day cycle 31, 30, 31, 30, 31 (we don't care about February +// because yday only ever count the days _before_ February 1, +// since February is the last month). +// +// Using the rule for deriving f and f° from f*, these multiplications +// convert to these divisions: +// +// century := (4 days + 3) / 146097 +// cdays := (4 days + 3) % 146097 / 4 +// cyear := (4 cdays + 3) / 1461 +// ayday := (4 cdays + 3) % 1461 / 4 +// amonth := (5 ayday + 461) / 153 +// mday := (5 ayday + 461) % 153 / 5 +// +// The a in ayday and amonth stands for absolute (March 1-based) +// to distinguish from the standard yday (January 1-based). +// +// After computing these, we can translate from the March 1 calendar +// to the standard January 1 calendar with branch-free math assuming a +// branch-free conversion from bool to int 0 or 1, denoted int(b) here: +// +// isJanFeb := int(yday >= marchThruDecember) +// month := amonth - isJanFeb*12 +// year := century*100 + cyear + isJanFeb +// isLeap := int(cyear%4 == 0) & (int(cyear != 0) | int(century%4 == 0)) +// day := 1 + mday +// yday := 1 + ayday + 31 + 28 + isLeap&^isJanFeb - 365*isJanFeb +// +// isLeap is the standard leap-year rule, but the split year form +// makes the divisions all reduce to binary masking. +// Note that day and yday are 1-based, in contrast to mday and ayday. +// To keep the various units separate, we define integer types +// for each. These are never stored in interfaces nor allocated, +// so their type information does not appear in Go binaries. const ( - // The unsigned zero year for internal calculations. - // Must be 1 mod 400, and times before it will not compute correctly, - // but otherwise can be changed at will. - absoluteZeroYear = -292277022399 + secondsPerMinute = 60 + secondsPerHour = 60 * secondsPerMinute + secondsPerDay = 24 * secondsPerHour + secondsPerWeek = 7 * secondsPerDay + daysPer400Years = 365*400 + 97 + + // Days from March 1 through end of year + marchThruDecember = 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31 + 30 + 31 + + // absoluteYears is the number of years we subtract from internal time to get absolute time. + // This value must be 0 mod 400, and it defines the “absolute zero instant” + // mentioned in the “Computations on Times” comment above: March 1, -absoluteYears. + // Dates before the absolute epoch will not compute correctly, + // but otherwise the value can be changed as needed. + absoluteYears = 292277022400 // The year of the zero Time. // Assumed by the unixToInternal computation below. internalYear = 1 // Offsets to convert between internal and absolute or Unix times. - absoluteToInternal int64 = (absoluteZeroYear - internalYear) * 365.2425 * secondsPerDay + absoluteToInternal int64 = -(absoluteYears*365.2425 + marchThruDecember) * secondsPerDay internalToAbsolute = -absoluteToInternal unixToInternal int64 = (1969*365 + 1969/4 - 1969/100 + 1969/400) * secondsPerDay internalToUnix int64 = -unixToInternal + absoluteToUnix = absoluteToInternal + internalToUnix + unixToAbsolute = unixToInternal + internalToAbsolute + wallToInternal int64 = (1884*365 + 1884/4 - 1884/100 + 1884/400) * secondsPerDay ) -// IsZero reports whether t represents the zero time instant, -// January 1, year 1, 00:00:00 UTC. -func (t Time) IsZero() bool { - return t.sec() == 0 && t.nsec() == 0 +// An absSeconds counts the number of seconds since the absolute zero instant. +type absSeconds uint64 + +// An absDays counts the number of days since the absolute zero instant. +type absDays uint64 + +// An absCentury counts the number of centuries since the absolute zero instant. +type absCentury uint64 + +// An absCyear counts the number of years since the start of a century. +type absCyear int + +// An absYday counts the number of days since the start of a year. +// Note that absolute years start on March 1. +type absYday int + +// An absMonth counts the number of months since the start of a year. +// absMonth=0 denotes March. +type absMonth int + +// An absLeap is a single bit (0 or 1) denoting whether a given year is a leap year. +type absLeap int + +// An absJanFeb is a single bit (0 or 1) denoting whether a given day falls in January or February. +// That is a special case because the absolute years start in March (unlike normal calendar years). +type absJanFeb int + +// dateToAbsDays takes a standard year/month/day and returns the +// number of days from the absolute epoch to that day. +// The days argument can be out of range and in particular can be negative. +func dateToAbsDays(year int64, month Month, day int) absDays { + // See “Computations on Times” comment above. + amonth := uint32(month) + janFeb := uint32(0) + if amonth < 3 { + janFeb = 1 + } + amonth += 12 * janFeb + y := uint64(year) - uint64(janFeb) + absoluteYears + + // For amonth is in the range [3,14], we want: + // + // ayday := (153*amonth - 457) / 5 + // + // (See the “Computations on Times” comment above + // as well as Neri and Schneider, section 7.) + // + // That is equivalent to: + // + // ayday := (979*amonth - 2919) >> 5 + // + // and the latter form uses a couple fewer instructions, + // so use it, saving a few cycles. + // See Neri and Schneider, section 8.3 + // for more about this optimization. + // + // (Note that there is no saved division, because the compiler + // implements / 5 without division in all cases.) + ayday := (979*amonth - 2919) >> 5 + + century := y / 100 + cyear := uint32(y % 100) + cday := 1461 * cyear / 4 + centurydays := 146097 * century / 4 + + return absDays(centurydays + uint64(int64(cday+ayday)+int64(day)-1)) } -// abs returns the time t as an absolute time, adjusted by the zone offset. +// days converts absolute seconds to absolute days. +func (abs absSeconds) days() absDays { + return absDays(abs / secondsPerDay) +} + +// split splits days into century, cyear, ayday. +func (days absDays) split() (century absCentury, cyear absCyear, ayday absYday) { + // See “Computations on Times” comment above. + d := 4*uint64(days) + 3 + century = absCentury(d / 146097) + + // This should be + // cday := uint32(d % 146097) / 4 + // cd := 4*cday + 3 + // which is to say + // cday := uint32(d % 146097) >> 2 + // cd := cday<<2 + 3 + // but of course (x>>2<<2)+3 == x|3, + // so do that instead. + cd := uint32(d%146097) | 3 + + // For cdays in the range [0,146097] (100 years), we want: + // + // cyear := (4 cdays + 3) / 1461 + // yday := (4 cdays + 3) % 1461 / 4 + // + // (See the “Computations on Times” comment above + // as well as Neri and Schneider, section 7.) + // + // That is equivalent to: + // + // cyear := (2939745 cdays) >> 32 + // yday := (2939745 cdays) & 0xFFFFFFFF / 2939745 / 4 + // + // so do that instead, saving a few cycles. + // See Neri and Schneider, section 8.3 + // for more about this optimization. + hi, lo := bits.Mul32(2939745, uint32(cd)) + cyear = absCyear(hi) + ayday = absYday(lo / 2939745 / 4) + return +} + +// split splits ayday into absolute month and standard (1-based) day-in-month. +func (ayday absYday) split() (m absMonth, mday int) { + // See “Computations on Times” comment above. + // + // For yday in the range [0,366], + // + // amonth := (5 yday + 461) / 153 + // mday := (5 yday + 461) % 153 / 5 + // + // is equivalent to: + // + // amonth = (2141 yday + 197913) >> 16 + // mday = (2141 yday + 197913) & 0xFFFF / 2141 + // + // so do that instead, saving a few cycles. + // See Neri and Schneider, section 8.3. + d := 2141*uint32(ayday) + 197913 + return absMonth(d >> 16), 1 + int((d&0xFFFF)/2141) +} + +// janFeb returns 1 if the March 1-based ayday is in January or February, 0 otherwise. +func (ayday absYday) janFeb() absJanFeb { + // See “Computations on Times” comment above. + jf := absJanFeb(0) + if ayday >= marchThruDecember { + jf = 1 + } + return jf +} + +// month returns the standard Month for (m, janFeb) +func (m absMonth) month(janFeb absJanFeb) Month { + // See “Computations on Times” comment above. + return Month(m) - Month(janFeb)*12 +} + +// leap returns 1 if (century, cyear) is a leap year, 0 otherwise. +func (century absCentury) leap(cyear absCyear) absLeap { + // See “Computations on Times” comment above. + y4ok := 0 + if cyear%4 == 0 { + y4ok = 1 + } + y100ok := 0 + if cyear != 0 { + y100ok = 1 + } + y400ok := 0 + if century%4 == 0 { + y400ok = 1 + } + return absLeap(y4ok & (y100ok | y400ok)) +} + +// year returns the standard year for (century, cyear, janFeb). +func (century absCentury) year(cyear absCyear, janFeb absJanFeb) int { + // See “Computations on Times” comment above. + return int(uint64(century)*100-absoluteYears) + int(cyear) + int(janFeb) +} + +// yday returns the standard 1-based yday for (ayday, janFeb, leap). +func (ayday absYday) yday(janFeb absJanFeb, leap absLeap) int { + // See “Computations on Times” comment above. + return int(ayday) + (1 + 31 + 28) + int(leap)&^int(janFeb) - 365*int(janFeb) +} + +// date converts days into standard year, month, day. +func (days absDays) date() (year int, month Month, day int) { + century, cyear, ayday := days.split() + amonth, day := ayday.split() + janFeb := ayday.janFeb() + year = century.year(cyear, janFeb) + month = amonth.month(janFeb) + return +} + +// yearYday converts days into the standard year and 1-based yday. +func (days absDays) yearYday() (year, yday int) { + century, cyear, ayday := days.split() + janFeb := ayday.janFeb() + year = century.year(cyear, janFeb) + yday = ayday.yday(janFeb, century.leap(cyear)) + return +} + +// absSec returns the time t as an absolute seconds, adjusted by the zone offset. // It is called when computing a presentation property like Month or Hour. -func (t Time) abs() uint64 { +// We'd rather call it abs, but there are linknames to abs that make that problematic. +// See timeAbs below. +func (t Time) absSec() absSeconds { l := t.loc // Avoid function calls when possible. if l == nil || l == &localLoc { @@ -483,12 +777,12 @@ func (t Time) abs() uint64 { sec += int64(offset) } } - return uint64(sec + (unixToInternal + internalToAbsolute)) + return absSeconds(sec + (unixToInternal + internalToAbsolute)) } // locabs is a combination of the Zone and abs methods, // extracting both return values from a single zone lookup. -func (t Time) locabs() (name string, offset int, abs uint64) { +func (t Time) locabs() (name string, offset int, abs absSeconds) { l := t.loc if l == nil || l == &localLoc { l = l.get() @@ -506,44 +800,45 @@ func (t Time) locabs() (name string, offset int, abs uint64) { } else { name = "UTC" } - abs = uint64(sec + (unixToInternal + internalToAbsolute)) + abs = absSeconds(sec + (unixToInternal + internalToAbsolute)) return } // Date returns the year, month, and day in which t occurs. func (t Time) Date() (year int, month Month, day int) { - year, month, day, _ = t.date(true) - return + return t.absSec().days().date() } // Year returns the year in which t occurs. func (t Time) Year() int { - year, _, _, _ := t.date(false) - return year + century, cyear, ayday := t.absSec().days().split() + janFeb := ayday.janFeb() + return century.year(cyear, janFeb) } // Month returns the month of the year specified by t. func (t Time) Month() Month { - _, month, _, _ := t.date(true) - return month + _, _, ayday := t.absSec().days().split() + amonth, _ := ayday.split() + return amonth.month(ayday.janFeb()) } // Day returns the day of the month specified by t. func (t Time) Day() int { - _, _, day, _ := t.date(true) + _, _, ayday := t.absSec().days().split() + _, day := ayday.split() return day } // Weekday returns the day of the week specified by t. func (t Time) Weekday() Weekday { - return absWeekday(t.abs()) + return t.absSec().days().weekday() } -// absWeekday is like Weekday but operates on an absolute time. -func absWeekday(abs uint64) Weekday { - // January 1 of the absolute year, like January 1 of 2001, was a Monday. - sec := (abs + uint64(Monday)*secondsPerDay) % secondsPerWeek - return Weekday(int(sec) / secondsPerDay) +// weekday returns the day of the week specified by days. +func (days absDays) weekday() Weekday { + // March 1 of the absolute year, like March 1 of 2000, was a Wednesday. + return Weekday((uint64(days) + uint64(Wednesday)) % 7) } // ISOWeek returns the ISO 8601 year and week number in which t occurs. @@ -561,35 +856,19 @@ func (t Time) ISOWeek() (year, week int) { // 1 2 3 4 5 6 7 // +3 +2 +1 0 -1 -2 -3 // the offset to Thursday - abs := t.abs() - d := Thursday - absWeekday(abs) - // handle Sunday - if d == 4 { - d = -3 - } - // find the Thursday of the calendar week - abs += uint64(d) * secondsPerDay - year, _, _, yday := absDate(abs, false) - return year, yday/7 + 1 + days := t.absSec().days() + thu := days + absDays(Thursday-((days-1).weekday()+1)) + year, yday := thu.yearYday() + return year, (yday-1)/7 + 1 } // Clock returns the hour, minute, and second within the day specified by t. func (t Time) Clock() (hour, min, sec int) { - return absClock(t.abs()) + return t.absSec().clock() } -// absClock is like clock but operates on an absolute time. -// -// absClock should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/phuslu/log -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname absClock -func absClock(abs uint64) (hour, min, sec int) { +// clock returns the hour, minute, and second within the day specified by abs. +func (abs absSeconds) clock() (hour, min, sec int) { sec = int(abs % secondsPerDay) hour = sec / secondsPerHour sec -= hour * secondsPerHour @@ -600,17 +879,17 @@ func absClock(abs uint64) (hour, min, sec int) { // Hour returns the hour within the day specified by t, in the range [0, 23]. func (t Time) Hour() int { - return int(t.abs()%secondsPerDay) / secondsPerHour + return int(t.absSec()%secondsPerDay) / secondsPerHour } // Minute returns the minute offset within the hour specified by t, in the range [0, 59]. func (t Time) Minute() int { - return int(t.abs()%secondsPerHour) / secondsPerMinute + return int(t.absSec()%secondsPerHour) / secondsPerMinute } // Second returns the second offset within the minute specified by t, in the range [0, 59]. func (t Time) Second() int { - return int(t.abs() % secondsPerMinute) + return int(t.absSec() % secondsPerMinute) } // Nanosecond returns the nanosecond offset within the second specified by t, @@ -622,8 +901,8 @@ func (t Time) Nanosecond() int { // YearDay returns the day of the year specified by t, in the range [1,365] for non-leap years, // and [1,366] in leap years. func (t Time) YearDay() int { - _, _, _, yday := t.date(false) - return yday + 1 + _, yday := t.absSec().days().yearYday() + return yday } // A Duration represents the elapsed time between two instants @@ -870,7 +1149,8 @@ func (d Duration) Round(m Duration) Duration { } // Abs returns the absolute value of d. -// As a special case, [math.MinInt64] is converted to [math.MaxInt64]. +// As a special case, Duration([math.MinInt64]) is converted to Duration([math.MaxInt64]), +// reducing its magnitude by 1 nanosecond. func (d Duration) Abs() Duration { switch { case d >= 0: @@ -981,167 +1261,68 @@ func (t Time) AddDate(years int, months int, days int) Time { return Date(year+years, month+Month(months), day+days, hour, min, sec, int(t.nsec()), t.Location()) } -const ( - secondsPerMinute = 60 - secondsPerHour = 60 * secondsPerMinute - secondsPerDay = 24 * secondsPerHour - secondsPerWeek = 7 * secondsPerDay - daysPer400Years = 365*400 + 97 - daysPer100Years = 365*100 + 24 - daysPer4Years = 365*4 + 1 -) - -// date computes the year, day of year, and when full=true, -// the month and day in which t occurs. -func (t Time) date(full bool) (year int, month Month, day int, yday int) { - return absDate(t.abs(), full) -} - -// absDate is like date but operates on an absolute time. -// -// absDate should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/phuslu/log -// - gitee.com/quant1x/gox -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname absDate -func absDate(abs uint64, full bool) (year int, month Month, day int, yday int) { - // Split into time and day. - d := abs / secondsPerDay - - // Account for 400 year cycles. - n := d / daysPer400Years - y := 400 * n - d -= daysPer400Years * n - - // Cut off 100-year cycles. - // The last cycle has one extra leap year, so on the last day - // of that year, day / daysPer100Years will be 4 instead of 3. - // Cut it back down to 3 by subtracting n>>2. - n = d / daysPer100Years - n -= n >> 2 - y += 100 * n - d -= daysPer100Years * n - - // Cut off 4-year cycles. - // The last cycle has a missing leap year, which does not - // affect the computation. - n = d / daysPer4Years - y += 4 * n - d -= daysPer4Years * n - - // Cut off years within a 4-year cycle. - // The last year is a leap year, so on the last day of that year, - // day / 365 will be 4 instead of 3. Cut it back down to 3 - // by subtracting n>>2. - n = d / 365 - n -= n >> 2 - y += n - d -= 365 * n - - year = int(int64(y) + absoluteZeroYear) - yday = int(d) - - if !full { - return +// daysBefore returns the number of days in a non-leap year before month m. +// daysBefore(December+1) returns 365. +func daysBefore(m Month) int { + adj := 0 + if m >= March { + adj = -2 } - day = yday - if isLeap(year) { - // Leap year - switch { - case day > 31+29-1: - // After leap day; pretend it wasn't there. - day-- - case day == 31+29-1: - // Leap day. - month = February - day = 29 - return - } - } - - // Estimate month on assumption that every month has 31 days. - // The estimate may be too low by at most one month, so adjust. - month = Month(day / 31) - end := int(daysBefore[month+1]) - var begin int - if day >= end { - month++ - begin = end - } else { - begin = int(daysBefore[month]) - } - - month++ // because January is 1 - day = day - begin + 1 - return -} - -// daysBefore[m] counts the number of days in a non-leap year -// before month m begins. There is an entry for m=12, counting -// the number of days before January of next year (365). -var daysBefore = [...]int32{ - 0, - 31, - 31 + 28, - 31 + 28 + 31, - 31 + 28 + 31 + 30, - 31 + 28 + 31 + 30 + 31, - 31 + 28 + 31 + 30 + 31 + 30, - 31 + 28 + 31 + 30 + 31 + 30 + 31, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31 + 30, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31 + 30 + 31, + // With the -2 adjustment after February, + // we need to compute the running sum of: + // 0 31 30 31 30 31 30 31 31 30 31 30 31 + // which is: + // 0 31 61 92 122 153 183 214 245 275 306 336 367 + // This is almost exactly 367/12×(m-1) except for the + // occasonal off-by-one suggesting there may be an + // integer approximation of the form (a×m + b)/c. + // A brute force search over small a, b, c finds that + // (214×m - 211) / 7 computes the function perfectly. + return (214*int(m)-211)/7 + adj } func daysIn(m Month, year int) int { - if m == February && isLeap(year) { - return 29 + if m == February { + if isLeap(year) { + return 29 + } + return 28 } - return int(daysBefore[m] - daysBefore[m-1]) -} - -// daysSinceEpoch takes a year and returns the number of days from -// the absolute epoch to the start of that year. -// This is basically (year - zeroYear) * 365, but accounting for leap days. -func daysSinceEpoch(year int) uint64 { - y := uint64(int64(year) - absoluteZeroYear) - - // Add in days from 400-year cycles. - n := y / 400 - y -= 400 * n - d := daysPer400Years * n - - // Add in 100-year cycles. - n = y / 100 - y -= 100 * n - d += daysPer100Years * n - - // Add in 4-year cycles. - n = y / 4 - y -= 4 * n - d += daysPer4Years * n - - // Add in non-leap years. - n = y - d += 365 * n - - return d + // With the special case of February eliminated, the pattern is + // 31 30 31 30 31 30 31 31 30 31 30 31 + // Adding m&1 produces the basic alternation; + // adding (m>>3)&1 inverts the alternation starting in August. + return 30 + int((m+m>>3)&1) } // Provided by package runtime. +// +// now returns the current real time, and is superseded by runtimeNow which returns +// the fake synctest clock when appropriate. +// +// now should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/phuslu/log +// - github.com/sethvargo/go-limiter +// - github.com/ulule/limiter/v3 +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. func now() (sec int64, nsec int32, mono int64) -// runtimeNano returns the current value of the runtime clock in nanoseconds. +// runtimeNow returns the current time. +// When called within a synctest.Run bubble, it returns the group's fake clock. // -//go:linkname runtimeNano runtime.nanotime +//go:linkname runtimeNow +func runtimeNow() (sec int64, nsec int32, mono int64) + +// runtimeNano returns the current value of the runtime clock in nanoseconds. +// When called within a synctest.Run bubble, it returns the group's fake clock. +// +//go:linkname runtimeNano func runtimeNano() int64 // Monotonic times are reported as offsets from startNano. @@ -1157,7 +1338,10 @@ var startNano int64 = runtimeNano() - 1 // Now returns the current local time. func Now() Time { - sec, nsec, mono := now() + sec, nsec, mono := runtimeNow() + if mono == 0 { + return Time{uint64(nsec), sec + unixToInternal, Local} + } mono -= startNano sec += unixToInternal - minWall if uint64(sec)>>33 != 0 { @@ -1275,8 +1459,8 @@ const ( timeBinaryVersionV2 // For LMT only ) -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (t Time) MarshalBinary() ([]byte, error) { +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (t Time) AppendBinary(b []byte) ([]byte, error) { var offsetMin int16 // minutes east of UTC. -1 is UTC. var offsetSec int8 version := timeBinaryVersionV1 @@ -1292,38 +1476,46 @@ func (t Time) MarshalBinary() ([]byte, error) { offset /= 60 if offset < -32768 || offset == -1 || offset > 32767 { - return nil, errors.New("Time.MarshalBinary: unexpected zone offset") + return b, errors.New("Time.MarshalBinary: unexpected zone offset") } offsetMin = int16(offset) } sec := t.sec() nsec := t.nsec() - enc := []byte{ - version, // byte 0 : version - byte(sec >> 56), // bytes 1-8: seconds - byte(sec >> 48), - byte(sec >> 40), - byte(sec >> 32), - byte(sec >> 24), - byte(sec >> 16), - byte(sec >> 8), + b = append(b, + version, // byte 0 : version + byte(sec>>56), // bytes 1-8: seconds + byte(sec>>48), + byte(sec>>40), + byte(sec>>32), + byte(sec>>24), + byte(sec>>16), + byte(sec>>8), byte(sec), - byte(nsec >> 24), // bytes 9-12: nanoseconds - byte(nsec >> 16), - byte(nsec >> 8), + byte(nsec>>24), // bytes 9-12: nanoseconds + byte(nsec>>16), + byte(nsec>>8), byte(nsec), - byte(offsetMin >> 8), // bytes 13-14: zone offset in minutes + byte(offsetMin>>8), // bytes 13-14: zone offset in minutes byte(offsetMin), - } + ) if version == timeBinaryVersionV2 { - enc = append(enc, byte(offsetSec)) + b = append(b, byte(offsetSec)) } - - return enc, nil + return b, nil } -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// MarshalBinary implements the [encoding.BinaryMarshaler] interface. +func (t Time) MarshalBinary() ([]byte, error) { + b, err := t.AppendBinary(make([]byte, 0, 16)) + if err != nil { + return nil, err + } + return b, nil +} + +// UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. func (t *Time) UnmarshalBinary(data []byte) error { buf := data if len(buf) == 0 { @@ -1385,7 +1577,7 @@ func (t *Time) GobDecode(data []byte) error { return t.UnmarshalBinary(data) } -// MarshalJSON implements the [json.Marshaler] interface. +// MarshalJSON implements the [encoding/json.Marshaler] interface. // The time is a quoted string in the RFC 3339 format with sub-second precision. // If the timestamp cannot be represented as valid RFC 3339 // (e.g., the year is out of range), then an error is reported. @@ -1400,7 +1592,7 @@ func (t Time) MarshalJSON() ([]byte, error) { return b, nil } -// UnmarshalJSON implements the [json.Unmarshaler] interface. +// UnmarshalJSON implements the [encoding/json.Unmarshaler] interface. // The time must be a quoted string in the RFC 3339 format. func (t *Time) UnmarshalJSON(data []byte) error { if string(data) == "null" { @@ -1416,19 +1608,30 @@ func (t *Time) UnmarshalJSON(data []byte) error { return err } -// MarshalText implements the [encoding.TextMarshaler] interface. -// The time is formatted in RFC 3339 format with sub-second precision. -// If the timestamp cannot be represented as valid RFC 3339 -// (e.g., the year is out of range), then an error is reported. -func (t Time) MarshalText() ([]byte, error) { - b := make([]byte, 0, len(RFC3339Nano)) +func (t Time) appendTo(b []byte, errPrefix string) ([]byte, error) { b, err := t.appendStrictRFC3339(b) if err != nil { - return nil, errors.New("Time.MarshalText: " + err.Error()) + return nil, errors.New(errPrefix + err.Error()) } return b, nil } +// AppendText implements the [encoding.TextAppender] interface. +// The time is formatted in RFC 3339 format with sub-second precision. +// If the timestamp cannot be represented as valid RFC 3339 +// (e.g., the year is out of range), then an error is returned. +func (t Time) AppendText(b []byte) ([]byte, error) { + return t.appendTo(b, "Time.AppendText: ") +} + +// MarshalText implements the [encoding.TextMarshaler] interface. The output +// matches that of calling the [Time.AppendText] method. +// +// See [Time.AppendText] for more information. +func (t Time) MarshalText() ([]byte, error) { + return t.appendTo(make([]byte, 0, len(RFC3339Nano)), "Time.MarshalText: ") +} + // UnmarshalText implements the [encoding.TextUnmarshaler] interface. // The time must be in the RFC 3339 format. func (t *Time) UnmarshalText(data []byte) error { @@ -1474,7 +1677,15 @@ func (t Time) IsDST() bool { } func isLeap(year int) bool { - return year%4 == 0 && (year%100 != 0 || year%400 == 0) + // year%4 == 0 && (year%100 != 0 || year%400 == 0) + // Bottom 2 bits must be clear. + // For multiples of 25, bottom 4 bits must be clear. + // Thanks to Cassio Neri for this trick. + mask := 0xf + if year%25 != 0 { + mask = 3 + } + return year&mask == 0 } // norm returns nhi, nlo such that @@ -1529,23 +1740,10 @@ func Date(year int, month Month, day, hour, min, sec, nsec int, loc *Location) T hour, min = norm(hour, min, 60) day, hour = norm(day, hour, 24) - // Compute days since the absolute epoch. - d := daysSinceEpoch(year) - - // Add in days before this month. - d += uint64(daysBefore[month-1]) - if isLeap(year) && month >= March { - d++ // February 29 - } - - // Add in days before today. - d += uint64(day - 1) - - // Add in time elapsed today. - abs := d * secondsPerDay - abs += uint64(hour*secondsPerHour + min*secondsPerMinute + sec) - - unix := int64(abs) + (absoluteToInternal + internalToUnix) + // Convert to absolute time and then Unix time. + unix := int64(dateToAbsDays(int64(year), month, day))*secondsPerDay + + int64(hour*secondsPerHour+min*secondsPerMinute+sec) + + absoluteToUnix // Look for zone offset for expected time, so we can adjust to UTC. // The lookup function expects UTC, so first we pass unix in the @@ -1693,3 +1891,50 @@ func div(t Time, d Duration) (qmod2 int, r Duration) { } return } + +// Regrettable Linkname Compatibility +// +// timeAbs, absDate, and absClock mimic old internal details, no longer used. +// Widely used packages linknamed these to get “faster” time routines. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/phuslu/log +// +// phuslu hard-coded 'Unix time + 9223372028715321600' [sic] +// as the input to absDate and absClock, using the old Jan 1-based +// absolute times. +// quant1x linknamed the time.Time.abs method and passed the +// result of that method to absDate and absClock. +// +// Keeping both of these working forces us to provide these three +// routines here, operating on the old Jan 1-based epoch instead +// of the new March 1-based epoch. And the fact that time.Time.abs +// was linknamed means that we have to call the current abs method +// something different (time.Time.absSec, defined above) to make it +// possible to provide this simulation of the old routines here. +// +// None of this code is linked into the binary if not referenced by +// these linkname-happy packages. In particular, despite its name, +// time.Time.abs does not appear in the time.Time method table. +// +// Do not remove these routines or their linknames, or change the +// type signature or meaning of arguments. + +//go:linkname legacyTimeTimeAbs time.Time.abs +func legacyTimeTimeAbs(t Time) uint64 { + return uint64(t.absSec() - marchThruDecember*secondsPerDay) +} + +//go:linkname legacyAbsClock time.absClock +func legacyAbsClock(abs uint64) (hour, min, sec int) { + return absSeconds(abs + marchThruDecember*secondsPerDay).clock() +} + +//go:linkname legacyAbsDate time.absDate +func legacyAbsDate(abs uint64, full bool) (year int, month Month, day int, yday int) { + d := absSeconds(abs + marchThruDecember*secondsPerDay).days() + year, month, day = d.date() + _, yday = d.yearYday() + yday-- // yearYday is 1-based, old API was 0-based + return +} diff --git a/src/time/time_test.go b/src/time/time_test.go index c12b9117..ff253be4 100644 --- a/src/time/time_test.go +++ b/src/time/time_test.go @@ -22,6 +22,26 @@ import ( . "time" ) +func TestInternal(t *testing.T) { + for _, tt := range InternalTests { + t.Run(tt.Name, func(t *testing.T) { tt.Test(t) }) + } +} + +func TestZeroTime(t *testing.T) { + var zero Time + year, month, day := zero.Date() + hour, min, sec := zero.Clock() + nsec := zero.Nanosecond() + yday := zero.YearDay() + wday := zero.Weekday() + if year != 1 || month != January || day != 1 || hour != 0 || min != 0 || sec != 0 || nsec != 0 || yday != 1 || wday != Monday { + t.Errorf("zero time = %v %v %v year %v %02d:%02d:%02d.%09d yday %d want Monday Jan 1 year 1 00:00:00.000000000 yday 1", + wday, month, day, year, hour, min, sec, nsec, yday) + } + +} + // We should be in PST/PDT, but if the time zone files are missing we // won't be. The purpose of this test is to at least explain why some of // the subsequent tests fail. @@ -103,75 +123,75 @@ func same(t Time, u *parsedTime) bool { t.Weekday() == u.Weekday } -func TestSecondsToUTC(t *testing.T) { +func TestUnixUTC(t *testing.T) { for _, test := range utctests { sec := test.seconds golden := &test.golden tm := Unix(sec, 0).UTC() newsec := tm.Unix() if newsec != sec { - t.Errorf("SecondsToUTC(%d).Seconds() = %d", sec, newsec) + t.Errorf("Unix(%d, 0).Unix() = %d", sec, newsec) } if !same(tm, golden) { - t.Errorf("SecondsToUTC(%d): // %#v", sec, tm) + t.Errorf("Unix(%d, 0): // %#v", sec, tm) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%v", tm.Format(RFC3339+" MST")) } } } -func TestNanosecondsToUTC(t *testing.T) { +func TestUnixNanoUTC(t *testing.T) { for _, test := range nanoutctests { golden := &test.golden nsec := test.seconds*1e9 + int64(golden.Nanosecond) tm := Unix(0, nsec).UTC() newnsec := tm.Unix()*1e9 + int64(tm.Nanosecond()) if newnsec != nsec { - t.Errorf("NanosecondsToUTC(%d).Nanoseconds() = %d", nsec, newnsec) + t.Errorf("Unix(0, %d).Nanoseconds() = %d", nsec, newnsec) } if !same(tm, golden) { - t.Errorf("NanosecondsToUTC(%d):", nsec) + t.Errorf("Unix(0, %d):", nsec) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%+v", tm.Format(RFC3339+" MST")) } } } -func TestSecondsToLocalTime(t *testing.T) { +func TestUnix(t *testing.T) { for _, test := range localtests { sec := test.seconds golden := &test.golden tm := Unix(sec, 0) newsec := tm.Unix() if newsec != sec { - t.Errorf("SecondsToLocalTime(%d).Seconds() = %d", sec, newsec) + t.Errorf("Unix(%d, 0).Seconds() = %d", sec, newsec) } if !same(tm, golden) { - t.Errorf("SecondsToLocalTime(%d):", sec) + t.Errorf("Unix(%d, 0):", sec) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%+v", tm.Format(RFC3339+" MST")) } } } -func TestNanosecondsToLocalTime(t *testing.T) { +func TestUnixNano(t *testing.T) { for _, test := range nanolocaltests { golden := &test.golden nsec := test.seconds*1e9 + int64(golden.Nanosecond) tm := Unix(0, nsec) newnsec := tm.Unix()*1e9 + int64(tm.Nanosecond()) if newnsec != nsec { - t.Errorf("NanosecondsToLocalTime(%d).Seconds() = %d", nsec, newnsec) + t.Errorf("Unix(0, %d).Seconds() = %d", nsec, newnsec) } if !same(tm, golden) { - t.Errorf("NanosecondsToLocalTime(%d):", nsec) + t.Errorf("Unix(0, %d):", nsec) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%+v", tm.Format(RFC3339+" MST")) } } } -func TestSecondsToUTCAndBack(t *testing.T) { +func TestUnixUTCAndBack(t *testing.T) { f := func(sec int64) bool { return Unix(sec, 0).UTC().Unix() == sec } f32 := func(sec int32) bool { return f(int64(sec)) } cfg := &quick.Config{MaxCount: 10000} @@ -185,7 +205,7 @@ func TestSecondsToUTCAndBack(t *testing.T) { } } -func TestNanosecondsToUTCAndBack(t *testing.T) { +func TestUnixNanoUTCAndBack(t *testing.T) { f := func(nsec int64) bool { t := Unix(0, nsec).UTC() ns := t.Unix()*1e9 + int64(t.Nanosecond()) @@ -647,6 +667,9 @@ var dateTests = []struct { {2012, 1, -43, 7, 56, 35, 0, Local, 1321631795}, // Jan -52 7:56:35 2012 {2012, int(January - 2), 18, 7, 56, 35, 0, Local, 1321631795}, // (Jan-2) 18 7:56:35 2012 {2010, int(December + 11), 18, 7, 56, 35, 0, Local, 1321631795}, // (Dec+11) 18 7:56:35 2010 + {1970, 1, 15297, 7, 56, 35, 0, Local, 1321631795}, // large number of days + + {1970, 1, -25508, 0, 0, 0, 0, Local, -2203948800}, // negative Unix time } func TestDate(t *testing.T) { @@ -685,6 +708,13 @@ func TestAddDate(t *testing.T) { time, t1) } } + + t2 := Date(1899, 12, 31, 0, 0, 0, 0, UTC) + days := t2.Unix() / (24 * 60 * 60) + t3 := Unix(0, 0).AddDate(0, 0, int(days)) + if !t2.Equal(t3) { + t.Errorf("Adddate(0, 0, %d) = %v, want %v", days, t3, t2) + } } var daysInTests = []struct { @@ -886,6 +916,16 @@ func TestMarshalInvalidTimes(t *testing.T) { case err == nil || err.Error() != want: t.Errorf("(%v).MarshalText() error = %v, want %v", tt.time, err, want) } + + buf := make([]byte, 0, 64) + want = strings.ReplaceAll(tt.want, "MarshalJSON", "AppendText") + b, err = tt.time.AppendText(buf) + switch { + case b != nil: + t.Errorf("(%v).AppendText() = %q, want nil", tt.time, b) + case err == nil || err.Error() != want: + t.Errorf("(%v).AppendText() error = %v, want %v", tt.time, err, want) + } } } @@ -1360,7 +1400,7 @@ var defaultLocTests = []struct { {"Add", func(t1, t2 Time) bool { return t1.Add(Hour).Equal(t2.Add(Hour)) }}, {"Sub", func(t1, t2 Time) bool { return t1.Sub(t2) == t2.Sub(t1) }}, - //Original caus for this test case bug 15852 + // Original cause for this test case bug 15852 {"AddDate", func(t1, t2 Time) bool { return t1.AddDate(1991, 9, 3) == t2.AddDate(1991, 9, 3) }}, {"UTC", func(t1, t2 Time) bool { return t1.UTC() == t2.UTC() }}, @@ -1379,6 +1419,13 @@ var defaultLocTests = []struct { {"UnixMilli", func(t1, t2 Time) bool { return t1.UnixMilli() == t2.UnixMilli() }}, {"UnixMicro", func(t1, t2 Time) bool { return t1.UnixMicro() == t2.UnixMicro() }}, + {"AppendBinary", func(t1, t2 Time) bool { + buf1 := make([]byte, 4, 32) + buf2 := make([]byte, 4, 32) + a1, b1 := t1.AppendBinary(buf1) + a2, b2 := t2.AppendBinary(buf2) + return bytes.Equal(a1[4:], a2[4:]) && b1 == b2 + }}, {"MarshalBinary", func(t1, t2 Time) bool { a1, b1 := t1.MarshalBinary() a2, b2 := t2.MarshalBinary() @@ -1394,6 +1441,14 @@ var defaultLocTests = []struct { a2, b2 := t2.MarshalJSON() return bytes.Equal(a1, a2) && b1 == b2 }}, + {"AppendText", func(t1, t2 Time) bool { + maxCap := len(RFC3339Nano) + 4 + buf1 := make([]byte, 4, maxCap) + buf2 := make([]byte, 4, maxCap) + a1, b1 := t1.AppendText(buf1) + a2, b2 := t2.AppendText(buf2) + return bytes.Equal(a1[4:], a2[4:]) && b1 == b2 + }}, {"MarshalText", func(t1, t2 Time) bool { a1, b1 := t1.MarshalText() a2, b2 := t2.MarshalText() @@ -1486,6 +1541,13 @@ func BenchmarkMarshalText(b *testing.B) { } } +func BenchmarkMarshalBinary(b *testing.B) { + t := Now() + for i := 0; i < b.N; i++ { + t.MarshalBinary() + } +} + func BenchmarkParse(b *testing.B) { for i := 0; i < b.N; i++ { Parse(ANSIC, "Mon Jan 2 15:04:05 2006") diff --git a/src/time/zoneinfo.go b/src/time/zoneinfo.go index 0fe13630..aee0e540 100644 --- a/src/time/zoneinfo.go +++ b/src/time/zoneinfo.go @@ -331,14 +331,11 @@ func tzset(s string, lastTxSec, sec int64) (name string, offset int, start, end return "", 0, 0, 0, false, false } - year, _, _, yday := absDate(uint64(sec+unixToInternal+internalToAbsolute), false) - - ysec := int64(yday*secondsPerDay) + sec%secondsPerDay - - // Compute start of year in seconds since Unix epoch. - d := daysSinceEpoch(year) - abs := int64(d * secondsPerDay) - abs += absoluteToInternal + internalToUnix + // Compute start of year in seconds since Unix epoch, + // and seconds since then to get to sec. + year, yday := absSeconds(sec + unixToInternal + internalToAbsolute).days().yearYday() + ysec := int64((yday-1)*secondsPerDay) + sec%secondsPerDay + ystart := sec - ysec startSec := int64(tzruleTime(year, startRule, stdOffset)) endSec := int64(tzruleTime(year, endRule, dstOffset)) @@ -358,11 +355,11 @@ func tzset(s string, lastTxSec, sec int64) (name string, offset int, start, end // just the start and end of the year. That suffices for // the only caller that cares, which is Date. if ysec < startSec { - return stdName, stdOffset, abs, startSec + abs, stdIsDST, true + return stdName, stdOffset, ystart, startSec + ystart, stdIsDST, true } else if ysec >= endSec { - return stdName, stdOffset, endSec + abs, abs + 365*secondsPerDay, stdIsDST, true + return stdName, stdOffset, endSec + ystart, ystart + 365*secondsPerDay, stdIsDST, true } else { - return dstName, dstOffset, startSec + abs, endSec + abs, dstIsDST, true + return dstName, dstOffset, startSec + ystart, endSec + ystart, dstIsDST, true } } @@ -596,7 +593,7 @@ func tzruleTime(year int, r rule, off int) int { } d += 7 } - d += int(daysBefore[r.mon-1]) + d += int(daysBefore(Month(r.mon))) if isLeap(year) && r.mon > 2 { d++ } diff --git a/src/time/zoneinfo_abbrs_windows.go b/src/time/zoneinfo_abbrs_windows.go index 27831743..814d9443 100644 --- a/src/time/zoneinfo_abbrs_windows.go +++ b/src/time/zoneinfo_abbrs_windows.go @@ -36,7 +36,7 @@ var abbrs = map[string]abbr{ "Central Standard Time": {"CST", "CDT"}, // America/Chicago "Central Brazilian Standard Time": {"-04", "-04"}, // America/Cuiaba "Mountain Standard Time": {"MST", "MDT"}, // America/Denver - "Greenland Standard Time": {"-03", "-02"}, // America/Godthab + "Greenland Standard Time": {"-02", "-01"}, // America/Godthab "Turks And Caicos Standard Time": {"EST", "EDT"}, // America/Grand_Turk "Central America Standard Time": {"CST", "CST"}, // America/Guatemala "Atlantic Standard Time": {"AST", "ADT"}, // America/Halifax @@ -58,13 +58,13 @@ var abbrs = map[string]abbr{ "Newfoundland Standard Time": {"NST", "NDT"}, // America/St_Johns "Pacific Standard Time (Mexico)": {"PST", "PDT"}, // America/Tijuana "Yukon Standard Time": {"MST", "MST"}, // America/Whitehorse - "Central Asia Standard Time": {"+06", "+06"}, // Asia/Almaty "Jordan Standard Time": {"+03", "+03"}, // Asia/Amman "Arabic Standard Time": {"+03", "+03"}, // Asia/Baghdad "Azerbaijan Standard Time": {"+04", "+04"}, // Asia/Baku "SE Asia Standard Time": {"+07", "+07"}, // Asia/Bangkok "Altai Standard Time": {"+07", "+07"}, // Asia/Barnaul "Middle East Standard Time": {"EET", "EEST"}, // Asia/Beirut + "Central Asia Standard Time": {"+06", "+06"}, // Asia/Bishkek "India Standard Time": {"IST", "IST"}, // Asia/Calcutta "Transbaikal Standard Time": {"+09", "+09"}, // Asia/Chita "Sri Lanka Standard Time": {"+0530", "+0530"}, // Asia/Colombo diff --git a/src/time/zoneinfo_plan9.go b/src/time/zoneinfo_plan9.go index d13b623a..036f669d 100644 --- a/src/time/zoneinfo_plan9.go +++ b/src/time/zoneinfo_plan9.go @@ -96,7 +96,7 @@ func loadZoneDataPlan9(s string) (l *Location, err error) { // Fill in the cache with information about right now, // since that will be the most common lookup. - sec, _, _ := now() + sec, _, _ := runtimeNow() for i := range tx { if tx[i].when <= sec && (i+1 == len(tx) || sec < tx[i+1].when) { l.cacheStart = tx[i].when diff --git a/src/time/zoneinfo_read.go b/src/time/zoneinfo_read.go index 5314b6ff..047e360d 100644 --- a/src/time/zoneinfo_read.go +++ b/src/time/zoneinfo_read.go @@ -320,7 +320,7 @@ func LoadLocationFromTZData(name string, data []byte) (*Location, error) { // Fill in the cache with information about right now, // since that will be the most common lookup. - sec, _, _ := now() + sec, _, _ := runtimeNow() for i := range tx { if tx[i].when <= sec && (i+1 == len(tx) || sec < tx[i+1].when) { l.cacheStart = tx[i].when diff --git a/src/time/zoneinfo_windows.go b/src/time/zoneinfo_windows.go index c9f38ea3..55b21fa3 100644 --- a/src/time/zoneinfo_windows.go +++ b/src/time/zoneinfo_windows.go @@ -20,7 +20,7 @@ var platformZoneSources []string // none: Windows uses system calls instead // time apply to all previous and future years as well. // matchZoneKey checks if stdname and dstname match the corresponding key -// values "MUI_Std" and MUI_Dlt" or "Std" and "Dlt" in the kname key stored +// values "MUI_Std" and "MUI_Dlt" or "Std" and "Dlt" in the kname key stored // under the open registry key zones. func matchZoneKey(zones registry.Key, kname string, stdname, dstname string) (matched bool, err2 error) { k, err := registry.OpenKey(zones, kname, registry.READ) diff --git a/src/unicode/letter.go b/src/unicode/letter.go index 9e2cead6..3959314c 100644 --- a/src/unicode/letter.go +++ b/src/unicode/letter.go @@ -206,34 +206,17 @@ func IsTitle(r rune) bool { return isExcludingLatin(Title, r) } -// to maps the rune using the specified case mapping. -// It additionally reports whether caseRange contained a mapping for r. -func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) { - if _case < 0 || MaxCase <= _case { - return ReplacementChar, false // as reasonable an error as any - } +// lookupCaseRange returns the CaseRange mapping for rune r or nil if no +// mapping exists for r. +func lookupCaseRange(r rune, caseRange []CaseRange) *CaseRange { // binary search over ranges lo := 0 hi := len(caseRange) for lo < hi { m := int(uint(lo+hi) >> 1) - cr := caseRange[m] + cr := &caseRange[m] if rune(cr.Lo) <= r && r <= rune(cr.Hi) { - delta := cr.Delta[_case] - if delta > MaxRune { - // In an Upper-Lower sequence, which always starts with - // an UpperCase letter, the real deltas always look like: - // {0, 1, 0} UpperCase (Lower is next) - // {-1, 0, -1} LowerCase (Upper, Title are previous) - // The characters at even offsets from the beginning of the - // sequence are upper case; the ones at odd offsets are lower. - // The correct mapping can be done by clearing or setting the low - // bit in the sequence offset. - // The constants UpperCase and TitleCase are even while LowerCase - // is odd so we take the low bit from _case. - return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)), true - } - return r + delta, true + return cr } if r < rune(cr.Lo) { hi = m @@ -241,6 +224,37 @@ func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping lo = m + 1 } } + return nil +} + +// convertCase converts r to _case using CaseRange cr. +func convertCase(_case int, r rune, cr *CaseRange) rune { + delta := cr.Delta[_case] + if delta > MaxRune { + // In an Upper-Lower sequence, which always starts with + // an UpperCase letter, the real deltas always look like: + // {0, 1, 0} UpperCase (Lower is next) + // {-1, 0, -1} LowerCase (Upper, Title are previous) + // The characters at even offsets from the beginning of the + // sequence are upper case; the ones at odd offsets are lower. + // The correct mapping can be done by clearing or setting the low + // bit in the sequence offset. + // The constants UpperCase and TitleCase are even while LowerCase + // is odd so we take the low bit from _case. + return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)) + } + return r + delta +} + +// to maps the rune using the specified case mapping. +// It additionally reports whether caseRange contained a mapping for r. +func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) { + if _case < 0 || MaxCase <= _case { + return ReplacementChar, false // as reasonable an error as any + } + if cr := lookupCaseRange(r, caseRange); cr != nil { + return convertCase(_case, r, cr), true + } return r, false } @@ -364,8 +378,11 @@ func SimpleFold(r rune) rune { // No folding specified. This is a one- or two-element // equivalence class containing rune and ToLower(rune) // and ToUpper(rune) if they are different from rune. - if l := ToLower(r); l != r { - return l + if cr := lookupCaseRange(r, CaseRanges); cr != nil { + if l := convertCase(LowerCase, r, cr); l != r { + return l + } + return convertCase(UpperCase, r, cr) } - return ToUpper(r) + return r } diff --git a/src/unicode/letter_test.go b/src/unicode/letter_test.go index 123f9a64..75c8aeee 100644 --- a/src/unicode/letter_test.go +++ b/src/unicode/letter_test.go @@ -642,3 +642,29 @@ func TestNegativeRune(t *testing.T) { } } } + +func BenchmarkToUpper(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = ToUpper('δ') + } +} + +func BenchmarkToLower(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = ToLower('Δ') + } +} + +func BenchmarkSimpleFold(b *testing.B) { + bench := func(name string, r rune) { + b.Run(name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = SimpleFold(r) + } + }) + } + bench("Upper", 'Δ') + bench("Lower", 'δ') + bench("Fold", '\u212A') + bench("NoFold", '習') +} diff --git a/src/unicode/utf16/utf16_test.go b/src/unicode/utf16/utf16_test.go index 74a4a674..3d434275 100644 --- a/src/unicode/utf16/utf16_test.go +++ b/src/unicode/utf16/utf16_test.go @@ -6,7 +6,7 @@ package utf16_test import ( "internal/testenv" - "reflect" + "slices" "testing" "unicode" . "unicode/utf16" @@ -58,7 +58,7 @@ var encodeTests = []encodeTest{ func TestEncode(t *testing.T) { for _, tt := range encodeTests { out := Encode(tt.in) - if !reflect.DeepEqual(out, tt.out) { + if !slices.Equal(out, tt.out) { t.Errorf("Encode(%x) = %x; want %x", tt.in, out, tt.out) } } @@ -70,7 +70,7 @@ func TestAppendRune(t *testing.T) { for _, u := range tt.in { out = AppendRune(out, u) } - if !reflect.DeepEqual(out, tt.out) { + if !slices.Equal(out, tt.out) { t.Errorf("AppendRune(%x) = %x; want %x", tt.in, out, tt.out) } } @@ -143,7 +143,7 @@ func TestAllocationsDecode(t *testing.T) { func TestDecode(t *testing.T) { for _, tt := range decodeTests { out := Decode(tt.in) - if !reflect.DeepEqual(out, tt.out) { + if !slices.Equal(out, tt.out) { t.Errorf("Decode(%x) = %x; want %x", tt.in, out, tt.out) } } diff --git a/src/unicode/utf8/utf8.go b/src/unicode/utf8/utf8.go index c7389d4d..180c008e 100644 --- a/src/unicode/utf8/utf8.go +++ b/src/unicode/utf8/utf8.go @@ -61,6 +61,12 @@ const ( s7 = 0x44 // accept 4, size 4 ) +const ( + runeErrorByte0 = t3 | (RuneError >> 12) + runeErrorByte1 = tx | (RuneError>>6)&maskx + runeErrorByte2 = tx | RuneError&maskx +) + // first is information about the first byte in a UTF-8 sequence. var first = [256]uint8{ // 1 2 3 4 5 6 7 8 9 A B C D E F @@ -340,32 +346,41 @@ func RuneLen(r rune) int { // If the rune is out of range, it writes the encoding of [RuneError]. // It returns the number of bytes written. func EncodeRune(p []byte, r rune) int { - // Negative values are erroneous. Making it unsigned addresses the problem. - switch i := uint32(r); { - case i <= rune1Max: + // This function is inlineable for fast handling of ASCII. + if uint32(r) <= rune1Max { p[0] = byte(r) return 1 + } + return encodeRuneNonASCII(p, r) +} + +func encodeRuneNonASCII(p []byte, r rune) int { + // Negative values are erroneous. Making it unsigned addresses the problem. + switch i := uint32(r); { case i <= rune2Max: _ = p[1] // eliminate bounds checks p[0] = t2 | byte(r>>6) p[1] = tx | byte(r)&maskx return 2 - case i > MaxRune, surrogateMin <= i && i <= surrogateMax: - r = RuneError - fallthrough - case i <= rune3Max: + case i < surrogateMin, surrogateMax < i && i <= rune3Max: _ = p[2] // eliminate bounds checks p[0] = t3 | byte(r>>12) p[1] = tx | byte(r>>6)&maskx p[2] = tx | byte(r)&maskx return 3 - default: + case i > rune3Max && i <= MaxRune: _ = p[3] // eliminate bounds checks p[0] = t4 | byte(r>>18) p[1] = tx | byte(r>>12)&maskx p[2] = tx | byte(r>>6)&maskx p[3] = tx | byte(r)&maskx return 4 + default: + _ = p[2] // eliminate bounds checks + p[0] = runeErrorByte0 + p[1] = runeErrorByte1 + p[2] = runeErrorByte2 + return 3 } } @@ -385,13 +400,12 @@ func appendRuneNonASCII(p []byte, r rune) []byte { switch i := uint32(r); { case i <= rune2Max: return append(p, t2|byte(r>>6), tx|byte(r)&maskx) - case i > MaxRune, surrogateMin <= i && i <= surrogateMax: - r = RuneError - fallthrough - case i <= rune3Max: + case i < surrogateMin, surrogateMax < i && i <= rune3Max: return append(p, t3|byte(r>>12), tx|byte(r>>6)&maskx, tx|byte(r)&maskx) - default: + case i > rune3Max && i <= MaxRune: return append(p, t4|byte(r>>18), tx|byte(r>>12)&maskx, tx|byte(r>>6)&maskx, tx|byte(r)&maskx) + default: + return append(p, runeErrorByte0, runeErrorByte1, runeErrorByte2) } } @@ -400,70 +414,19 @@ func appendRuneNonASCII(p []byte, r rune) []byte { func RuneCount(p []byte) int { np := len(p) var n int - for i := 0; i < np; { - n++ - c := p[i] - if c < RuneSelf { - // ASCII fast path - i++ - continue + for ; n < np; n++ { + if c := p[n]; c >= RuneSelf { + // non-ASCII slow path + return n + RuneCountInString(string(p[n:])) } - x := first[c] - if x == xx { - i++ // invalid. - continue - } - size := int(x & 7) - if i+size > np { - i++ // Short or invalid. - continue - } - accept := acceptRanges[x>>4] - if c := p[i+1]; c < accept.lo || accept.hi < c { - size = 1 - } else if size == 2 { - } else if c := p[i+2]; c < locb || hicb < c { - size = 1 - } else if size == 3 { - } else if c := p[i+3]; c < locb || hicb < c { - size = 1 - } - i += size } return n } // RuneCountInString is like [RuneCount] but its input is a string. func RuneCountInString(s string) (n int) { - ns := len(s) - for i := 0; i < ns; n++ { - c := s[i] - if c < RuneSelf { - // ASCII fast path - i++ - continue - } - x := first[c] - if x == xx { - i++ // invalid. - continue - } - size := int(x & 7) - if i+size > ns { - i++ // Short or invalid. - continue - } - accept := acceptRanges[x>>4] - if c := s[i+1]; c < accept.lo || accept.hi < c { - size = 1 - } else if size == 2 { - } else if c := s[i+2]; c < locb || hicb < c { - size = 1 - } else if size == 3 { - } else if c := s[i+3]; c < locb || hicb < c { - size = 1 - } - i += size + for range s { + n++ } return n } diff --git a/src/unicode/utf8/utf8_test.go b/src/unicode/utf8/utf8_test.go index 19a04dc9..69362d2c 100644 --- a/src/unicode/utf8/utf8_test.go +++ b/src/unicode/utf8/utf8_test.go @@ -170,7 +170,7 @@ func TestDecodeRune(t *testing.T) { } r, size = DecodeRune(b[0 : len(b)-1]) if r != RuneError || size != wantsize { - t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b[0:len(b)-1], r, size, RuneError, wantsize) + t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b[:len(b)-1], r, size, RuneError, wantsize) } s = m.str[0 : len(m.str)-1] r, size = DecodeRuneInString(s) @@ -438,6 +438,15 @@ func TestRuneCount(t *testing.T) { } } +func TestRuneCountNonASCIIAllocation(t *testing.T) { + if n := testing.AllocsPerRun(10, func() { + s := []byte("日本語日本語日本語日") + _ = RuneCount(s) + }); n > 0 { + t.Errorf("unexpected RuneCount allocation, got %v, want 0", n) + } +} + type RuneLenTest struct { r rune size int @@ -641,28 +650,98 @@ func init() { func BenchmarkEncodeASCIIRune(b *testing.B) { buf := make([]byte, UTFMax) for i := 0; i < b.N; i++ { - EncodeRune(buf, 'a') + EncodeRune(buf, 'a') // 1 byte + } +} + +func BenchmarkEncodeSpanishRune(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + EncodeRune(buf, 'Ñ') // 2 bytes } } func BenchmarkEncodeJapaneseRune(b *testing.B) { buf := make([]byte, UTFMax) for i := 0; i < b.N; i++ { - EncodeRune(buf, '本') + EncodeRune(buf, '本') // 3 bytes + } +} + +func BenchmarkEncodeMaxRune(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + EncodeRune(buf, MaxRune) // 4 bytes + } +} + +func BenchmarkEncodeInvalidRuneMaxPlusOne(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + EncodeRune(buf, MaxRune+1) // 3 bytes: RuneError + } +} + +func BenchmarkEncodeInvalidRuneSurrogate(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + EncodeRune(buf, 0xD800) // 3 bytes: RuneError + } +} + +func BenchmarkEncodeInvalidRuneNegative(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + EncodeRune(buf, -1) // 3 bytes: RuneError } } func BenchmarkAppendASCIIRune(b *testing.B) { buf := make([]byte, UTFMax) for i := 0; i < b.N; i++ { - AppendRune(buf[:0], 'a') + AppendRune(buf[:0], 'a') // 1 byte + } +} + +func BenchmarkAppendSpanishRune(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + AppendRune(buf[:0], 'Ñ') // 2 bytes } } func BenchmarkAppendJapaneseRune(b *testing.B) { buf := make([]byte, UTFMax) for i := 0; i < b.N; i++ { - AppendRune(buf[:0], '本') + AppendRune(buf[:0], '本') // 3 bytes + } +} + +func BenchmarkAppendMaxRune(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + AppendRune(buf[:0], MaxRune) // 4 bytes + } +} + +func BenchmarkAppendInvalidRuneMaxPlusOne(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + AppendRune(buf[:0], MaxRune+1) // 3 bytes: RuneError + } +} + +func BenchmarkAppendInvalidRuneSurrogate(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + AppendRune(buf[:0], 0xD800) // 3 bytes: RuneError + } +} + +func BenchmarkAppendInvalidRuneNegative(b *testing.B) { + buf := make([]byte, UTFMax) + for i := 0; i < b.N; i++ { + AppendRune(buf[:0], -1) // 3 bytes: RuneError } } diff --git a/src/unique/handle.go b/src/unique/handle.go index abc620f6..520ab70f 100644 --- a/src/unique/handle.go +++ b/src/unique/handle.go @@ -6,13 +6,15 @@ package unique import ( "internal/abi" - "internal/concurrent" - "internal/weak" + isync "internal/sync" "runtime" "sync" - _ "unsafe" + "unsafe" + "weak" ) +var zero uintptr + // Handle is a globally unique identity for some value of type T. // // Two handles compare equal exactly if the two values used to create the handles @@ -23,15 +25,20 @@ type Handle[T comparable] struct { } // Value returns a shallow copy of the T value that produced the Handle. +// Value is safe for concurrent use by multiple goroutines. func (h Handle[T]) Value() T { return *h.value } // Make returns a globally unique handle for a value of type T. Handles // are equal if and only if the values used to produce them are equal. +// Make is safe for concurrent use by multiple goroutines. func Make[T comparable](value T) Handle[T] { // Find the map for type T. typ := abi.TypeFor[T]() + if typ.Size() == 0 { + return Handle[T]{(*T)(unsafe.Pointer(&zero))} + } ma, ok := uniqueMaps.Load(typ) if !ok { // This is a good time to initialize cleanup, since we must go through @@ -69,7 +76,7 @@ func Make[T comparable](value T) Handle[T] { } // Now that we're sure there's a value in the map, let's // try to get the pointer we need out of it. - ptr = wp.Strong() + ptr = wp.Value() if ptr != nil { break } @@ -91,7 +98,7 @@ var ( // benefit of not cramming every different type into a single map, but that's certainly // not enough to outweigh the cost of two map lookups. What is worth it though, is saving // on those allocations. - uniqueMaps = concurrent.NewHashTrieMap[*abi.Type, any]() // any is always a *uniqueMap[T]. + uniqueMaps isync.HashTrieMap[*abi.Type, any] // any is always a *uniqueMap[T]. // cleanupFuncs are functions that clean up dead weak pointers in type-specific // maps in uniqueMaps. We express cleanup this way because there's no way to iterate @@ -107,7 +114,7 @@ var ( ) type uniqueMap[T comparable] struct { - *concurrent.HashTrieMap[T, weak.Pointer[T]] + isync.HashTrieMap[T, weak.Pointer[T]] cloneSeq } @@ -116,10 +123,7 @@ func addUniqueMap[T comparable](typ *abi.Type) *uniqueMap[T] { // race with someone else, but that's fine; it's one // small, stray allocation. The number of allocations // this can create is bounded by a small constant. - m := &uniqueMap[T]{ - HashTrieMap: concurrent.NewHashTrieMap[T, weak.Pointer[T]](), - cloneSeq: makeCloneSeq(typ), - } + m := &uniqueMap[T]{cloneSeq: makeCloneSeq(typ)} a, loaded := uniqueMaps.LoadOrStore(typ, m) if !loaded { // Add a cleanup function for the new map. @@ -128,7 +132,7 @@ func addUniqueMap[T comparable](typ *abi.Type) *uniqueMap[T] { // Delete all the entries whose weak references are nil and clean up // deleted entries. m.All()(func(key T, wp weak.Pointer[T]) bool { - if wp.Strong() == nil { + if wp.Value() == nil { m.CompareAndDelete(key, wp) } return true diff --git a/src/unique/handle_test.go b/src/unique/handle_test.go index dd4b01ef..4b708760 100644 --- a/src/unique/handle_test.go +++ b/src/unique/handle_test.go @@ -9,6 +9,7 @@ import ( "internal/abi" "reflect" "runtime" + "strconv" "strings" "testing" "time" @@ -31,20 +32,22 @@ type testStruct struct { z float64 b string } +type testZeroSize struct{} func TestHandle(t *testing.T) { - testHandle[testString](t, "foo") - testHandle[testString](t, "bar") - testHandle[testString](t, "") - testHandle[testIntArray](t, [4]int{7, 77, 777, 7777}) - testHandle[testEface](t, nil) - testHandle[testStringArray](t, [3]string{"a", "b", "c"}) - testHandle[testStringStruct](t, testStringStruct{"x"}) - testHandle[testStringStructArrayStruct](t, testStringStructArrayStruct{ - s: [2]testStringStruct{testStringStruct{"y"}, testStringStruct{"z"}}, + testHandle(t, testString("foo")) + testHandle(t, testString("bar")) + testHandle(t, testString("")) + testHandle(t, testIntArray{7, 77, 777, 7777}) + testHandle(t, testEface(nil)) + testHandle(t, testStringArray{"a", "b", "c"}) + testHandle(t, testStringStruct{"x"}) + testHandle(t, testStringStructArrayStruct{ + s: [2]testStringStruct{{"y"}, {"z"}}, }) - testHandle[testStruct](t, testStruct{0.5, "184"}) - testHandle[testEface](t, testEface("hello")) + testHandle(t, testStruct{0.5, "184"}) + testHandle(t, testEface("hello")) + testHandle(t, testZeroSize(struct{}{})) } func testHandle[T comparable](t *testing.T, value T) { @@ -65,15 +68,19 @@ func testHandle[T comparable](t *testing.T, value T) { t.Error("v0 != v1") } - drainMaps(t) + drainMaps[T](t) checkMapsFor(t, value) }) } // drainMaps ensures that the internal maps are drained. -func drainMaps(t *testing.T) { +func drainMaps[T comparable](t *testing.T) { t.Helper() + if unsafe.Sizeof(*(new(T))) == 0 { + return // zero-size types are not inserted. + } + wait := make(chan struct{}, 1) // Set up a one-time notification for the next time the cleanup runs. @@ -107,7 +114,7 @@ func checkMapsFor[T comparable](t *testing.T, value T) { if !ok { return } - if wp.Strong() != nil { + if wp.Value() != nil { t.Errorf("value %v still referenced a handle (or tiny block?) ", value) return } @@ -132,3 +139,26 @@ func TestMakeClonesStrings(t *testing.T) { } runtime.KeepAlive(h) } + +func TestHandleUnsafeString(t *testing.T) { + var testData []string + for i := range 1024 { + testData = append(testData, strconv.Itoa(i)) + } + var buf []byte + var handles []Handle[string] + for _, s := range testData { + if len(buf) < len(s) { + buf = make([]byte, len(s)*2) + } + copy(buf, s) + sbuf := unsafe.String(&buf[0], len(s)) + handles = append(handles, Make(sbuf)) + } + for i, s := range testData { + h := Make(s) + if handles[i].Value() != h.Value() { + t.Fatal("unsafe string improperly retained internally") + } + } +} diff --git a/src/vendor/golang.org/x/crypto/LICENSE b/src/vendor/golang.org/x/crypto/LICENSE index 6a66aea5..2a7cf70d 100644 --- a/src/vendor/golang.org/x/crypto/LICENSE +++ b/src/vendor/golang.org/x/crypto/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go b/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go index db42e667..c709b728 100644 --- a/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go +++ b/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!arm64 && !s390x && !ppc64le) || !gc || purego +//go:build (!arm64 && !s390x && !ppc64 && !ppc64le) || !gc || purego package chacha20 diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go similarity index 89% rename from src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go rename to src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go index 3a4287f9..bd183d9b 100644 --- a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go +++ b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build gc && !purego +//go:build gc && !purego && (ppc64 || ppc64le) package chacha20 diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s similarity index 76% rename from src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s rename to src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s index c672ccf6..a660b411 100644 --- a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s +++ b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s @@ -19,7 +19,7 @@ // The differences in this and the original implementation are // due to the calling conventions and initialization of constants. -//go:build gc && !purego +//go:build gc && !purego && (ppc64 || ppc64le) #include "textflag.h" @@ -36,32 +36,68 @@ // for VPERMXOR #define MASK R18 -DATA consts<>+0x00(SB)/8, $0x3320646e61707865 -DATA consts<>+0x08(SB)/8, $0x6b20657479622d32 -DATA consts<>+0x10(SB)/8, $0x0000000000000001 -DATA consts<>+0x18(SB)/8, $0x0000000000000000 -DATA consts<>+0x20(SB)/8, $0x0000000000000004 -DATA consts<>+0x28(SB)/8, $0x0000000000000000 -DATA consts<>+0x30(SB)/8, $0x0a0b08090e0f0c0d -DATA consts<>+0x38(SB)/8, $0x0203000106070405 -DATA consts<>+0x40(SB)/8, $0x090a0b080d0e0f0c -DATA consts<>+0x48(SB)/8, $0x0102030005060704 -DATA consts<>+0x50(SB)/8, $0x6170786561707865 -DATA consts<>+0x58(SB)/8, $0x6170786561707865 -DATA consts<>+0x60(SB)/8, $0x3320646e3320646e -DATA consts<>+0x68(SB)/8, $0x3320646e3320646e -DATA consts<>+0x70(SB)/8, $0x79622d3279622d32 -DATA consts<>+0x78(SB)/8, $0x79622d3279622d32 -DATA consts<>+0x80(SB)/8, $0x6b2065746b206574 -DATA consts<>+0x88(SB)/8, $0x6b2065746b206574 -DATA consts<>+0x90(SB)/8, $0x0000000100000000 -DATA consts<>+0x98(SB)/8, $0x0000000300000002 -DATA consts<>+0xa0(SB)/8, $0x5566774411223300 -DATA consts<>+0xa8(SB)/8, $0xddeeffcc99aabb88 -DATA consts<>+0xb0(SB)/8, $0x6677445522330011 -DATA consts<>+0xb8(SB)/8, $0xeeffccddaabb8899 +DATA consts<>+0x00(SB)/4, $0x61707865 +DATA consts<>+0x04(SB)/4, $0x3320646e +DATA consts<>+0x08(SB)/4, $0x79622d32 +DATA consts<>+0x0c(SB)/4, $0x6b206574 +DATA consts<>+0x10(SB)/4, $0x00000001 +DATA consts<>+0x14(SB)/4, $0x00000000 +DATA consts<>+0x18(SB)/4, $0x00000000 +DATA consts<>+0x1c(SB)/4, $0x00000000 +DATA consts<>+0x20(SB)/4, $0x00000004 +DATA consts<>+0x24(SB)/4, $0x00000000 +DATA consts<>+0x28(SB)/4, $0x00000000 +DATA consts<>+0x2c(SB)/4, $0x00000000 +DATA consts<>+0x30(SB)/4, $0x0e0f0c0d +DATA consts<>+0x34(SB)/4, $0x0a0b0809 +DATA consts<>+0x38(SB)/4, $0x06070405 +DATA consts<>+0x3c(SB)/4, $0x02030001 +DATA consts<>+0x40(SB)/4, $0x0d0e0f0c +DATA consts<>+0x44(SB)/4, $0x090a0b08 +DATA consts<>+0x48(SB)/4, $0x05060704 +DATA consts<>+0x4c(SB)/4, $0x01020300 +DATA consts<>+0x50(SB)/4, $0x61707865 +DATA consts<>+0x54(SB)/4, $0x61707865 +DATA consts<>+0x58(SB)/4, $0x61707865 +DATA consts<>+0x5c(SB)/4, $0x61707865 +DATA consts<>+0x60(SB)/4, $0x3320646e +DATA consts<>+0x64(SB)/4, $0x3320646e +DATA consts<>+0x68(SB)/4, $0x3320646e +DATA consts<>+0x6c(SB)/4, $0x3320646e +DATA consts<>+0x70(SB)/4, $0x79622d32 +DATA consts<>+0x74(SB)/4, $0x79622d32 +DATA consts<>+0x78(SB)/4, $0x79622d32 +DATA consts<>+0x7c(SB)/4, $0x79622d32 +DATA consts<>+0x80(SB)/4, $0x6b206574 +DATA consts<>+0x84(SB)/4, $0x6b206574 +DATA consts<>+0x88(SB)/4, $0x6b206574 +DATA consts<>+0x8c(SB)/4, $0x6b206574 +DATA consts<>+0x90(SB)/4, $0x00000000 +DATA consts<>+0x94(SB)/4, $0x00000001 +DATA consts<>+0x98(SB)/4, $0x00000002 +DATA consts<>+0x9c(SB)/4, $0x00000003 +DATA consts<>+0xa0(SB)/4, $0x11223300 +DATA consts<>+0xa4(SB)/4, $0x55667744 +DATA consts<>+0xa8(SB)/4, $0x99aabb88 +DATA consts<>+0xac(SB)/4, $0xddeeffcc +DATA consts<>+0xb0(SB)/4, $0x22330011 +DATA consts<>+0xb4(SB)/4, $0x66774455 +DATA consts<>+0xb8(SB)/4, $0xaabb8899 +DATA consts<>+0xbc(SB)/4, $0xeeffccdd GLOBL consts<>(SB), RODATA, $0xc0 +#ifdef GOARCH_ppc64 +#define BE_XXBRW_INIT() \ + LVSL (R0)(R0), V24 \ + VSPLTISB $3, V25 \ + VXOR V24, V25, V24 \ + +#define BE_XXBRW(vr) VPERM vr, vr, V24, vr +#else +#define BE_XXBRW_INIT() +#define BE_XXBRW(vr) +#endif + //func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32) TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40 MOVD out+0(FP), OUT @@ -94,6 +130,8 @@ TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40 // Clear V27 VXOR V27, V27, V27 + BE_XXBRW_INIT() + // V28 LXVW4X (CONSTBASE)(R11), VS60 @@ -299,6 +337,11 @@ loop_vsx: VADDUWM V8, V18, V8 VADDUWM V12, V19, V12 + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + CMPU LEN, $64 BLT tail_vsx @@ -327,6 +370,11 @@ loop_vsx: VADDUWM V9, V18, V8 VADDUWM V13, V19, V12 + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + CMPU LEN, $64 BLT tail_vsx @@ -334,8 +382,8 @@ loop_vsx: LXVW4X (INP)(R8), VS60 LXVW4X (INP)(R9), VS61 LXVW4X (INP)(R10), VS62 - VXOR V27, V0, V27 + VXOR V27, V0, V27 VXOR V28, V4, V28 VXOR V29, V8, V29 VXOR V30, V12, V30 @@ -354,6 +402,11 @@ loop_vsx: VADDUWM V10, V18, V8 VADDUWM V14, V19, V12 + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + CMPU LEN, $64 BLT tail_vsx @@ -381,6 +434,11 @@ loop_vsx: VADDUWM V11, V18, V8 VADDUWM V15, V19, V12 + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + CMPU LEN, $64 BLT tail_vsx @@ -408,9 +466,9 @@ loop_vsx: done_vsx: // Increment counter by number of 64 byte blocks - MOVD (CNT), R14 + MOVWZ (CNT), R14 ADD BLOCKS, R14 - MOVD R14, (CNT) + MOVWZ R14, (CNT) RET tail_vsx: diff --git a/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305.go b/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305.go index 93da7322..8cf5d811 100644 --- a/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305.go +++ b/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305.go @@ -5,7 +5,7 @@ // Package chacha20poly1305 implements the ChaCha20-Poly1305 AEAD and its // extended nonce variant XChaCha20-Poly1305, as specified in RFC 8439 and // draft-irtf-cfrg-xchacha-01. -package chacha20poly1305 // import "golang.org/x/crypto/chacha20poly1305" +package chacha20poly1305 import ( "crypto/cipher" diff --git a/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s b/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s index 731d2ac6..fd5ee845 100644 --- a/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s +++ b/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s @@ -1,2715 +1,9762 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file was originally from https://golang.org/cl/24717 by Vlad Krasnov of CloudFlare. +// Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT. //go:build gc && !purego #include "textflag.h" -// General register allocation -#define oup DI -#define inp SI -#define inl BX -#define adp CX // free to reuse, after we hash the additional data -#define keyp R8 // free to reuse, when we copy the key to stack -#define itr2 R9 // general iterator -#define itr1 CX // general iterator -#define acc0 R10 -#define acc1 R11 -#define acc2 R12 -#define t0 R13 -#define t1 R14 -#define t2 R15 -#define t3 R8 -// Register and stack allocation for the SSE code -#define rStore (0*16)(BP) -#define sStore (1*16)(BP) -#define state1Store (2*16)(BP) -#define state2Store (3*16)(BP) -#define tmpStore (4*16)(BP) -#define ctr0Store (5*16)(BP) -#define ctr1Store (6*16)(BP) -#define ctr2Store (7*16)(BP) -#define ctr3Store (8*16)(BP) -#define A0 X0 -#define A1 X1 -#define A2 X2 -#define B0 X3 -#define B1 X4 -#define B2 X5 -#define C0 X6 -#define C1 X7 -#define C2 X8 -#define D0 X9 -#define D1 X10 -#define D2 X11 -#define T0 X12 -#define T1 X13 -#define T2 X14 -#define T3 X15 -#define A3 T0 -#define B3 T1 -#define C3 T2 -#define D3 T3 -// Register and stack allocation for the AVX2 code -#define rsStoreAVX2 (0*32)(BP) -#define state1StoreAVX2 (1*32)(BP) -#define state2StoreAVX2 (2*32)(BP) -#define ctr0StoreAVX2 (3*32)(BP) -#define ctr1StoreAVX2 (4*32)(BP) -#define ctr2StoreAVX2 (5*32)(BP) -#define ctr3StoreAVX2 (6*32)(BP) -#define tmpStoreAVX2 (7*32)(BP) // 256 bytes on stack -#define AA0 Y0 -#define AA1 Y5 -#define AA2 Y6 -#define AA3 Y7 -#define BB0 Y14 -#define BB1 Y9 -#define BB2 Y10 -#define BB3 Y11 -#define CC0 Y12 -#define CC1 Y13 -#define CC2 Y8 -#define CC3 Y15 -#define DD0 Y4 -#define DD1 Y1 -#define DD2 Y2 -#define DD3 Y3 -#define TT0 DD3 -#define TT1 AA3 -#define TT2 BB3 -#define TT3 CC3 -// ChaCha20 constants -DATA ·chacha20Constants<>+0x00(SB)/4, $0x61707865 -DATA ·chacha20Constants<>+0x04(SB)/4, $0x3320646e -DATA ·chacha20Constants<>+0x08(SB)/4, $0x79622d32 -DATA ·chacha20Constants<>+0x0c(SB)/4, $0x6b206574 -DATA ·chacha20Constants<>+0x10(SB)/4, $0x61707865 -DATA ·chacha20Constants<>+0x14(SB)/4, $0x3320646e -DATA ·chacha20Constants<>+0x18(SB)/4, $0x79622d32 -DATA ·chacha20Constants<>+0x1c(SB)/4, $0x6b206574 -// <<< 16 with PSHUFB -DATA ·rol16<>+0x00(SB)/8, $0x0504070601000302 -DATA ·rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A -DATA ·rol16<>+0x10(SB)/8, $0x0504070601000302 -DATA ·rol16<>+0x18(SB)/8, $0x0D0C0F0E09080B0A -// <<< 8 with PSHUFB -DATA ·rol8<>+0x00(SB)/8, $0x0605040702010003 -DATA ·rol8<>+0x08(SB)/8, $0x0E0D0C0F0A09080B -DATA ·rol8<>+0x10(SB)/8, $0x0605040702010003 -DATA ·rol8<>+0x18(SB)/8, $0x0E0D0C0F0A09080B -DATA ·avx2InitMask<>+0x00(SB)/8, $0x0 -DATA ·avx2InitMask<>+0x08(SB)/8, $0x0 -DATA ·avx2InitMask<>+0x10(SB)/8, $0x1 -DATA ·avx2InitMask<>+0x18(SB)/8, $0x0 - -DATA ·avx2IncMask<>+0x00(SB)/8, $0x2 -DATA ·avx2IncMask<>+0x08(SB)/8, $0x0 -DATA ·avx2IncMask<>+0x10(SB)/8, $0x2 -DATA ·avx2IncMask<>+0x18(SB)/8, $0x0 -// Poly1305 key clamp -DATA ·polyClampMask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF -DATA ·polyClampMask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC -DATA ·polyClampMask<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF -DATA ·polyClampMask<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF - -DATA ·sseIncMask<>+0x00(SB)/8, $0x1 -DATA ·sseIncMask<>+0x08(SB)/8, $0x0 -// To load/store the last < 16 bytes in a buffer -DATA ·andMask<>+0x00(SB)/8, $0x00000000000000ff -DATA ·andMask<>+0x08(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x10(SB)/8, $0x000000000000ffff -DATA ·andMask<>+0x18(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x20(SB)/8, $0x0000000000ffffff -DATA ·andMask<>+0x28(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x30(SB)/8, $0x00000000ffffffff -DATA ·andMask<>+0x38(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x40(SB)/8, $0x000000ffffffffff -DATA ·andMask<>+0x48(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x50(SB)/8, $0x0000ffffffffffff -DATA ·andMask<>+0x58(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x60(SB)/8, $0x00ffffffffffffff -DATA ·andMask<>+0x68(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x70(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0x78(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x80(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0x88(SB)/8, $0x00000000000000ff -DATA ·andMask<>+0x90(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0x98(SB)/8, $0x000000000000ffff -DATA ·andMask<>+0xa0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xa8(SB)/8, $0x0000000000ffffff -DATA ·andMask<>+0xb0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xb8(SB)/8, $0x00000000ffffffff -DATA ·andMask<>+0xc0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xc8(SB)/8, $0x000000ffffffffff -DATA ·andMask<>+0xd0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xd8(SB)/8, $0x0000ffffffffffff -DATA ·andMask<>+0xe0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xe8(SB)/8, $0x00ffffffffffffff - -GLOBL ·chacha20Constants<>(SB), (NOPTR+RODATA), $32 -GLOBL ·rol16<>(SB), (NOPTR+RODATA), $32 -GLOBL ·rol8<>(SB), (NOPTR+RODATA), $32 -GLOBL ·sseIncMask<>(SB), (NOPTR+RODATA), $16 -GLOBL ·avx2IncMask<>(SB), (NOPTR+RODATA), $32 -GLOBL ·avx2InitMask<>(SB), (NOPTR+RODATA), $32 -GLOBL ·polyClampMask<>(SB), (NOPTR+RODATA), $32 -GLOBL ·andMask<>(SB), (NOPTR+RODATA), $240 -// No PALIGNR in Go ASM yet (but VPALIGNR is present). -#define shiftB0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X3, X3 -#define shiftB1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x04 // PALIGNR $4, X4, X4 -#define shiftB2Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X5, X5 -#define shiftB3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X13, X13 -#define shiftC0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X6, X6 -#define shiftC1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x08 // PALIGNR $8, X7, X7 -#define shiftC2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc0; BYTE $0x08 // PALIGNR $8, X8, X8 -#define shiftC3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X14, X14 -#define shiftD0Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x0c // PALIGNR $12, X9, X9 -#define shiftD1Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x0c // PALIGNR $12, X10, X10 -#define shiftD2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X11, X11 -#define shiftD3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x0c // PALIGNR $12, X15, X15 -#define shiftB0Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X3, X3 -#define shiftB1Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x0c // PALIGNR $12, X4, X4 -#define shiftB2Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X5, X5 -#define shiftB3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X13, X13 -#define shiftC0Right shiftC0Left -#define shiftC1Right shiftC1Left -#define shiftC2Right shiftC2Left -#define shiftC3Right shiftC3Left -#define shiftD0Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x04 // PALIGNR $4, X9, X9 -#define shiftD1Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x04 // PALIGNR $4, X10, X10 -#define shiftD2Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X11, X11 -#define shiftD3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x04 // PALIGNR $4, X15, X15 - -// Some macros - -// ROL rotates the uint32s in register R left by N bits, using temporary T. -#define ROL(N, R, T) \ - MOVO R, T; PSLLL $(N), T; PSRLL $(32-(N)), R; PXOR T, R - -// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed. -#ifdef GOAMD64_v2 -#define ROL16(R, T) PSHUFB ·rol16<>(SB), R -#else -#define ROL16(R, T) ROL(16, R, T) -#endif - -// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed. -#ifdef GOAMD64_v2 -#define ROL8(R, T) PSHUFB ·rol8<>(SB), R -#else -#define ROL8(R, T) ROL(8, R, T) -#endif - -#define chachaQR(A, B, C, D, T) \ - PADDD B, A; PXOR A, D; ROL16(D, T) \ - PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $12, T; PSRLL $20, B; PXOR T, B \ - PADDD B, A; PXOR A, D; ROL8(D, T) \ - PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $7, T; PSRLL $25, B; PXOR T, B - -#define chachaQR_AVX2(A, B, C, D, T) \ - VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol16<>(SB), D, D \ - VPADDD D, C, C; VPXOR C, B, B; VPSLLD $12, B, T; VPSRLD $20, B, B; VPXOR T, B, B \ - VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol8<>(SB), D, D \ - VPADDD D, C, C; VPXOR C, B, B; VPSLLD $7, B, T; VPSRLD $25, B, B; VPXOR T, B, B - -#define polyAdd(S) ADDQ S, acc0; ADCQ 8+S, acc1; ADCQ $1, acc2 -#define polyMulStage1 MOVQ (0*8)(BP), AX; MOVQ AX, t2; MULQ acc0; MOVQ AX, t0; MOVQ DX, t1; MOVQ (0*8)(BP), AX; MULQ acc1; IMULQ acc2, t2; ADDQ AX, t1; ADCQ DX, t2 -#define polyMulStage2 MOVQ (1*8)(BP), AX; MOVQ AX, t3; MULQ acc0; ADDQ AX, t1; ADCQ $0, DX; MOVQ DX, acc0; MOVQ (1*8)(BP), AX; MULQ acc1; ADDQ AX, t2; ADCQ $0, DX -#define polyMulStage3 IMULQ acc2, t3; ADDQ acc0, t2; ADCQ DX, t3 -#define polyMulReduceStage MOVQ t0, acc0; MOVQ t1, acc1; MOVQ t2, acc2; ANDQ $3, acc2; MOVQ t2, t0; ANDQ $-4, t0; MOVQ t3, t1; SHRQ $2, t3, t2; SHRQ $2, t3; ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $0, acc2; ADDQ t2, acc0; ADCQ t3, acc1; ADCQ $0, acc2 - -#define polyMulStage1_AVX2 MOVQ (0*8)(BP), DX; MOVQ DX, t2; MULXQ acc0, t0, t1; IMULQ acc2, t2; MULXQ acc1, AX, DX; ADDQ AX, t1; ADCQ DX, t2 -#define polyMulStage2_AVX2 MOVQ (1*8)(BP), DX; MULXQ acc0, acc0, AX; ADDQ acc0, t1; MULXQ acc1, acc1, t3; ADCQ acc1, t2; ADCQ $0, t3 -#define polyMulStage3_AVX2 IMULQ acc2, DX; ADDQ AX, t2; ADCQ DX, t3 - -#define polyMul polyMulStage1; polyMulStage2; polyMulStage3; polyMulReduceStage -#define polyMulAVX2 polyMulStage1_AVX2; polyMulStage2_AVX2; polyMulStage3_AVX2; polyMulReduceStage -// ---------------------------------------------------------------------------- +// func polyHashADInternal<>() TEXT polyHashADInternal<>(SB), NOSPLIT, $0 - // adp points to beginning of additional data - // itr2 holds ad length - XORQ acc0, acc0 - XORQ acc1, acc1 - XORQ acc2, acc2 - CMPQ itr2, $13 - JNE hashADLoop + // Hack: Must declare #define macros inside of a function due to Avo constraints + // ROL rotates the uint32s in register R left by N bits, using temporary T. + #define ROL(N, R, T) \ + MOVO R, T; \ + PSLLL $(N), T; \ + PSRLL $(32-(N)), R; \ + PXOR T, R -openFastTLSAD: - // Special treatment for the TLS case of 13 bytes - MOVQ (adp), acc0 - MOVQ 5(adp), acc1 - SHRQ $24, acc1 - MOVQ $1, acc2 - polyMul + // ROL8 rotates the uint32s in register R left by 8, using temporary T if needed. + #ifdef GOAMD64_v2 + #define ROL8(R, T) PSHUFB ·rol8<>(SB), R + #else + #define ROL8(R, T) ROL(8, R, T) + #endif + + // ROL16 rotates the uint32s in register R left by 16, using temporary T if needed. + #ifdef GOAMD64_v2 + #define ROL16(R, T) PSHUFB ·rol16<>(SB), R + #else + #define ROL16(R, T) ROL(16, R, T) + #endif + XORQ R10, R10 + XORQ R11, R11 + XORQ R12, R12 + CMPQ R9, $0x0d + JNE hashADLoop + MOVQ (CX), R10 + MOVQ 5(CX), R11 + SHRQ $0x18, R11 + MOVQ $0x00000001, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 RET hashADLoop: // Hash in 16 byte chunks - CMPQ itr2, $16 - JB hashADTail - polyAdd(0(adp)) - LEAQ (1*16)(adp), adp - SUBQ $16, itr2 - polyMul - JMP hashADLoop + CMPQ R9, $0x10 + JB hashADTail + ADDQ (CX), R10 + ADCQ 8(CX), R11 + ADCQ $0x01, R12 + LEAQ 16(CX), CX + SUBQ $0x10, R9 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + JMP hashADLoop hashADTail: - CMPQ itr2, $0 + CMPQ R9, $0x00 JE hashADDone // Hash last < 16 byte tail - XORQ t0, t0 - XORQ t1, t1 - XORQ t2, t2 - ADDQ itr2, adp + XORQ R13, R13 + XORQ R14, R14 + XORQ R15, R15 + ADDQ R9, CX hashADTailLoop: - SHLQ $8, t0, t1 - SHLQ $8, t0 - MOVB -1(adp), t2 - XORQ t2, t0 - DECQ adp - DECQ itr2 - JNE hashADTailLoop + SHLQ $0x08, R13, R14 + SHLQ $0x08, R13 + MOVB -1(CX), R15 + XORQ R15, R13 + DECQ CX + DECQ R9 + JNE hashADTailLoop + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 -hashADTailFinish: - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul - - // Finished AD hashADDone: RET -// ---------------------------------------------------------------------------- -// func chacha20Poly1305Open(dst, key, src, ad []byte) bool -TEXT ·chacha20Poly1305Open(SB), 0, $288-97 +// func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool +// Requires: AVX, AVX2, BMI2, CMOV, SSE2 +TEXT ·chacha20Poly1305Open(SB), $288-97 // For aligned stack access MOVQ SP, BP - ADDQ $32, BP + ADDQ $0x20, BP ANDQ $-32, BP - MOVQ dst+0(FP), oup - MOVQ key+24(FP), keyp - MOVQ src+48(FP), inp - MOVQ src_len+56(FP), inl - MOVQ ad+72(FP), adp + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX // Check for AVX2 support - CMPB ·useAVX2(SB), $1 + CMPB ·useAVX2+0(SB), $0x01 JE chacha20Poly1305Open_AVX2 // Special optimization, for very short buffers - CMPQ inl, $128 - JBE openSSE128 // About 16% faster + CMPQ BX, $0x80 + JBE openSSE128 // For long buffers, prepare the poly key first - MOVOU ·chacha20Constants<>(SB), A0 - MOVOU (1*16)(keyp), B0 - MOVOU (2*16)(keyp), C0 - MOVOU (3*16)(keyp), D0 - MOVO D0, T1 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 + MOVO X9, X13 // Store state on stack for future use - MOVO B0, state1Store - MOVO C0, state2Store - MOVO D0, ctr3Store - MOVQ $10, itr2 + MOVO X3, 32(BP) + MOVO X6, 48(BP) + MOVO X9, 128(BP) + MOVQ $0x0000000a, R9 openSSEPreparePolyKey: - chachaQR(A0, B0, C0, D0, T0) - shiftB0Left; shiftC0Left; shiftD0Left - chachaQR(A0, B0, C0, D0, T0) - shiftB0Right; shiftC0Right; shiftD0Right - DECQ itr2 - JNE openSSEPreparePolyKey + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + DECQ R9 + JNE openSSEPreparePolyKey // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0 + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL 32(BP), X3 // Clamp and store the key - PAND ·polyClampMask<>(SB), A0 - MOVO A0, rStore; MOVO B0, sStore + PAND ·polyClampMask<>+0(SB), X0 + MOVO X0, (BP) + MOVO X3, 16(BP) // Hash AAD - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) openSSEMainLoop: - CMPQ inl, $256 + CMPQ BX, $0x00000100 JB openSSEMainLoopDone // Load state, increment counter blocks - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) - // There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16 - MOVQ $4, itr1 - MOVQ inp, itr2 + // There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash + // 2 blocks, and for the remaining 4 only 1 block - for a total of 16 + MOVQ $0x00000004, CX + MOVQ SI, R9 openSSEInternalLoop: - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyAdd(0(itr2)) - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - polyMulStage1 - polyMulStage2 - LEAQ (2*8)(itr2), itr2 - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - polyMulStage3 - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyMulReduceStage - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - DECQ itr1 - JGE openSSEInternalLoop - - polyAdd(0(itr2)) - polyMul - LEAQ (2*8)(itr2), itr2 - - CMPQ itr1, $-6 - JG openSSEInternalLoop + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + LEAQ 16(R9), R9 + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + DECQ CX + JGE openSSEInternalLoop + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + CMPQ CX, $-6 + JG openSSEInternalLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X6 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 80(BP), X9 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 // Load - xor - store - MOVO D3, tmpStore - MOVOU (0*16)(inp), D3; PXOR D3, A0; MOVOU A0, (0*16)(oup) - MOVOU (1*16)(inp), D3; PXOR D3, B0; MOVOU B0, (1*16)(oup) - MOVOU (2*16)(inp), D3; PXOR D3, C0; MOVOU C0, (2*16)(oup) - MOVOU (3*16)(inp), D3; PXOR D3, D0; MOVOU D0, (3*16)(oup) - MOVOU (4*16)(inp), D0; PXOR D0, A1; MOVOU A1, (4*16)(oup) - MOVOU (5*16)(inp), D0; PXOR D0, B1; MOVOU B1, (5*16)(oup) - MOVOU (6*16)(inp), D0; PXOR D0, C1; MOVOU C1, (6*16)(oup) - MOVOU (7*16)(inp), D0; PXOR D0, D1; MOVOU D1, (7*16)(oup) - MOVOU (8*16)(inp), D0; PXOR D0, A2; MOVOU A2, (8*16)(oup) - MOVOU (9*16)(inp), D0; PXOR D0, B2; MOVOU B2, (9*16)(oup) - MOVOU (10*16)(inp), D0; PXOR D0, C2; MOVOU C2, (10*16)(oup) - MOVOU (11*16)(inp), D0; PXOR D0, D2; MOVOU D2, (11*16)(oup) - MOVOU (12*16)(inp), D0; PXOR D0, A3; MOVOU A3, (12*16)(oup) - MOVOU (13*16)(inp), D0; PXOR D0, B3; MOVOU B3, (13*16)(oup) - MOVOU (14*16)(inp), D0; PXOR D0, C3; MOVOU C3, (14*16)(oup) - MOVOU (15*16)(inp), D0; PXOR tmpStore, D0; MOVOU D0, (15*16)(oup) - LEAQ 256(inp), inp - LEAQ 256(oup), oup - SUBQ $256, inl + MOVO X15, 64(BP) + MOVOU (SI), X15 + PXOR X15, X0 + MOVOU X0, (DI) + MOVOU 16(SI), X15 + PXOR X15, X3 + MOVOU X3, 16(DI) + MOVOU 32(SI), X15 + PXOR X15, X6 + MOVOU X6, 32(DI) + MOVOU 48(SI), X15 + PXOR X15, X9 + MOVOU X9, 48(DI) + MOVOU 64(SI), X9 + PXOR X9, X1 + MOVOU X1, 64(DI) + MOVOU 80(SI), X9 + PXOR X9, X4 + MOVOU X4, 80(DI) + MOVOU 96(SI), X9 + PXOR X9, X7 + MOVOU X7, 96(DI) + MOVOU 112(SI), X9 + PXOR X9, X10 + MOVOU X10, 112(DI) + MOVOU 128(SI), X9 + PXOR X9, X2 + MOVOU X2, 128(DI) + MOVOU 144(SI), X9 + PXOR X9, X5 + MOVOU X5, 144(DI) + MOVOU 160(SI), X9 + PXOR X9, X8 + MOVOU X8, 160(DI) + MOVOU 176(SI), X9 + PXOR X9, X11 + MOVOU X11, 176(DI) + MOVOU 192(SI), X9 + PXOR X9, X12 + MOVOU X12, 192(DI) + MOVOU 208(SI), X9 + PXOR X9, X13 + MOVOU X13, 208(DI) + MOVOU 224(SI), X9 + PXOR X9, X14 + MOVOU X14, 224(DI) + MOVOU 240(SI), X9 + PXOR 64(BP), X9 + MOVOU X9, 240(DI) + LEAQ 256(SI), SI + LEAQ 256(DI), DI + SUBQ $0x00000100, BX JMP openSSEMainLoop openSSEMainLoopDone: // Handle the various tail sizes efficiently - TESTQ inl, inl + TESTQ BX, BX JE openSSEFinalize - CMPQ inl, $64 + CMPQ BX, $0x40 JBE openSSETail64 - CMPQ inl, $128 + CMPQ BX, $0x80 JBE openSSETail128 - CMPQ inl, $192 + CMPQ BX, $0xc0 JBE openSSETail192 JMP openSSETail256 openSSEFinalize: // Hash in the PT, AAD lengths - ADDQ ad_len+80(FP), acc0; ADCQ src_len+56(FP), acc1; ADCQ $1, acc2 - polyMul + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Final reduce - MOVQ acc0, t0 - MOVQ acc1, t1 - MOVQ acc2, t2 - SUBQ $-5, acc0 - SBBQ $-1, acc1 - SBBQ $3, acc2 - CMOVQCS t0, acc0 - CMOVQCS t1, acc1 - CMOVQCS t2, acc2 + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 // Add in the "s" part of the key - ADDQ 0+sStore, acc0 - ADCQ 8+sStore, acc1 + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 // Finally, constant time compare to the tag at the end of the message XORQ AX, AX - MOVQ $1, DX - XORQ (0*8)(inp), acc0 - XORQ (1*8)(inp), acc1 - ORQ acc1, acc0 + MOVQ $0x00000001, DX + XORQ (SI), R10 + XORQ 8(SI), R11 + ORQ R11, R10 CMOVQEQ DX, AX // Return true iff tags are equal MOVB AX, ret+96(FP) RET -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 129 bytes openSSE128: - // For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks - MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO B0, T1; MOVO C0, T2; MOVO D1, T3 - MOVQ $10, itr2 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X3, X13 + MOVO X6, X14 + MOVO X10, X15 + MOVQ $0x0000000a, R9 openSSE128InnerCipherLoop: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftB1Left; shiftB2Left - shiftC0Left; shiftC1Left; shiftC2Left - shiftD0Left; shiftD1Left; shiftD2Left - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftB1Right; shiftB2Right - shiftC0Right; shiftC1Right; shiftC2Right - shiftD0Right; shiftD1Right; shiftD2Right - DECQ itr2 - JNE openSSE128InnerCipherLoop + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + DECQ R9 + JNE openSSE128InnerCipherLoop // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL T1, B0; PADDL T1, B1; PADDL T1, B2 - PADDL T2, C1; PADDL T2, C2 - PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2 + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL X13, X3 + PADDL X13, X4 + PADDL X13, X5 + PADDL X14, X7 + PADDL X14, X8 + PADDL X15, X10 + PADDL ·sseIncMask<>+0(SB), X15 + PADDL X15, X11 // Clamp and store the key - PAND ·polyClampMask<>(SB), A0 - MOVOU A0, rStore; MOVOU B0, sStore + PAND ·polyClampMask<>+0(SB), X0 + MOVOU X0, (BP) + MOVOU X3, 16(BP) // Hash - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) openSSE128Open: - CMPQ inl, $16 + CMPQ BX, $0x10 JB openSSETail16 - SUBQ $16, inl + SUBQ $0x10, BX // Load for hashing - polyAdd(0(inp)) + ADDQ (SI), R10 + ADCQ 8(SI), R11 + ADCQ $0x01, R12 // Load for decryption - MOVOU (inp), T0; PXOR T0, A1; MOVOU A1, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup - polyMul + MOVOU (SI), X12 + PXOR X12, X1 + MOVOU X1, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Shift the stream "left" - MOVO B1, A1 - MOVO C1, B1 - MOVO D1, C1 - MOVO A2, D1 - MOVO B2, A2 - MOVO C2, B2 - MOVO D2, C2 + MOVO X4, X1 + MOVO X7, X4 + MOVO X10, X7 + MOVO X2, X10 + MOVO X5, X2 + MOVO X8, X5 + MOVO X11, X8 JMP openSSE128Open openSSETail16: - TESTQ inl, inl + TESTQ BX, BX JE openSSEFinalize // We can safely load the CT from the end, because it is padded with the MAC - MOVQ inl, itr2 - SHLQ $4, itr2 - LEAQ ·andMask<>(SB), t0 - MOVOU (inp), T0 - ADDQ inl, inp - PAND -16(t0)(itr2*1), T0 - MOVO T0, 0+tmpStore - MOVQ T0, t0 - MOVQ 8+tmpStore, t1 - PXOR A1, T0 + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVOU (SI), X12 + ADDQ BX, SI + PAND -16(R13)(R9*1), X12 + MOVO X12, 64(BP) + MOVQ X12, R13 + MOVQ 72(BP), R14 + PXOR X1, X12 // We can only store one byte at a time, since plaintext can be shorter than 16 bytes openSSETail16Store: - MOVQ T0, t3 - MOVB t3, (oup) - PSRLDQ $1, T0 - INCQ oup - DECQ inl + MOVQ X12, R8 + MOVB R8, (DI) + PSRLDQ $0x01, X12 + INCQ DI + DECQ BX JNE openSSETail16Store - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 JMP openSSEFinalize -// ---------------------------------------------------------------------------- -// Special optimization for the last 64 bytes of ciphertext openSSETail64: - // Need to decrypt up to 64 bytes - prepare single block - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store - XORQ itr2, itr2 - MOVQ inl, itr1 - CMPQ itr1, $16 - JB openSSETail64LoopB + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 80(BP) + XORQ R9, R9 + MOVQ BX, CX + CMPQ CX, $0x10 + JB openSSETail64LoopB openSSETail64LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMul - SUBQ $16, itr1 + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + SUBQ $0x10, CX openSSETail64LoopB: - ADDQ $16, itr2 - chachaQR(A0, B0, C0, D0, T0) - shiftB0Left; shiftC0Left; shiftD0Left - chachaQR(A0, B0, C0, D0, T0) - shiftB0Right; shiftC0Right; shiftD0Right - - CMPQ itr1, $16 - JAE openSSETail64LoopA - - CMPQ itr2, $160 - JNE openSSETail64LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0; PADDL state2Store, C0; PADDL ctr0Store, D0 + ADDQ $0x10, R9 + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + CMPQ CX, $0x10 + JAE openSSETail64LoopA + CMPQ R9, $0xa0 + JNE openSSETail64LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL 32(BP), X3 + PADDL 48(BP), X6 + PADDL 80(BP), X9 openSSETail64DecLoop: - CMPQ inl, $16 + CMPQ BX, $0x10 JB openSSETail64DecLoopDone - SUBQ $16, inl - MOVOU (inp), T0 - PXOR T0, A0 - MOVOU A0, (oup) - LEAQ 16(inp), inp - LEAQ 16(oup), oup - MOVO B0, A0 - MOVO C0, B0 - MOVO D0, C0 + SUBQ $0x10, BX + MOVOU (SI), X12 + PXOR X12, X0 + MOVOU X0, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + MOVO X3, X0 + MOVO X6, X3 + MOVO X9, X6 JMP openSSETail64DecLoop openSSETail64DecLoopDone: - MOVO A0, A1 + MOVO X0, X1 JMP openSSETail16 -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of ciphertext openSSETail128: - // Need to decrypt up to 128 bytes - prepare two blocks - MOVO ·chacha20Constants<>(SB), A1; MOVO state1Store, B1; MOVO state2Store, C1; MOVO ctr3Store, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr0Store - MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr1Store - XORQ itr2, itr2 - MOVQ inl, itr1 - ANDQ $-16, itr1 + MOVO ·chacha20Constants<>+0(SB), X1 + MOVO 32(BP), X4 + MOVO 48(BP), X7 + MOVO 128(BP), X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 80(BP) + MOVO X1, X0 + MOVO X4, X3 + MOVO X7, X6 + MOVO X10, X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 96(BP) + XORQ R9, R9 + MOVQ BX, CX + ANDQ $-16, CX openSSETail128LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMul + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openSSETail128LoopB: - ADDQ $16, itr2 - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right + ADDQ $0x10, R9 + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + CMPQ R9, CX + JB openSSETail128LoopA + CMPQ R9, $0xa0 + JNE openSSETail128LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 96(BP), X9 + PADDL 80(BP), X10 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X1 + PXOR X13, X4 + PXOR X14, X7 + PXOR X15, X10 + MOVOU X1, (DI) + MOVOU X4, 16(DI) + MOVOU X7, 32(DI) + MOVOU X10, 48(DI) + SUBQ $0x40, BX + LEAQ 64(SI), SI + LEAQ 64(DI), DI + JMP openSSETail64DecLoop - CMPQ itr2, itr1 - JB openSSETail128LoopA - - CMPQ itr2, $160 - JNE openSSETail128LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1 - PADDL state1Store, B0; PADDL state1Store, B1 - PADDL state2Store, C0; PADDL state2Store, C1 - PADDL ctr1Store, D0; PADDL ctr0Store, D1 - - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1 - MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup) - - SUBQ $64, inl - LEAQ 64(inp), inp - LEAQ 64(oup), oup - JMP openSSETail64DecLoop - -// ---------------------------------------------------------------------------- -// Special optimization for the last 192 bytes of ciphertext openSSETail192: - // Need to decrypt up to 192 bytes - prepare three blocks - MOVO ·chacha20Constants<>(SB), A2; MOVO state1Store, B2; MOVO state2Store, C2; MOVO ctr3Store, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr0Store - MOVO A2, A1; MOVO B2, B1; MOVO C2, C1; MOVO D2, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store - MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr2Store - - MOVQ inl, itr1 - MOVQ $160, itr2 - CMPQ itr1, $160 - CMOVQGT itr2, itr1 - ANDQ $-16, itr1 - XORQ itr2, itr2 + MOVO ·chacha20Constants<>+0(SB), X2 + MOVO 32(BP), X5 + MOVO 48(BP), X8 + MOVO 128(BP), X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X11, 80(BP) + MOVO X2, X1 + MOVO X5, X4 + MOVO X8, X7 + MOVO X11, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 96(BP) + MOVO X1, X0 + MOVO X4, X3 + MOVO X7, X6 + MOVO X10, X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 112(BP) + MOVQ BX, CX + MOVQ $0x000000a0, R9 + CMPQ CX, $0xa0 + CMOVQGT R9, CX + ANDQ $-16, CX + XORQ R9, R9 openSSLTail192LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMul + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openSSLTail192LoopB: - ADDQ $16, itr2 - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - shiftB2Left; shiftC2Left; shiftD2Left - - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right - shiftB2Right; shiftC2Right; shiftD2Right - - CMPQ itr2, itr1 - JB openSSLTail192LoopA - - CMPQ itr2, $160 - JNE openSSLTail192LoopB - - CMPQ inl, $176 - JB openSSLTail192Store - - polyAdd(160(inp)) - polyMul - - CMPQ inl, $192 - JB openSSLTail192Store - - polyAdd(176(inp)) - polyMul + ADDQ $0x10, R9 + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + CMPQ R9, CX + JB openSSLTail192LoopA + CMPQ R9, $0xa0 + JNE openSSLTail192LoopB + CMPQ BX, $0xb0 + JB openSSLTail192Store + ADDQ 160(SI), R10 + ADCQ 168(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + CMPQ BX, $0xc0 + JB openSSLTail192Store + ADDQ 176(SI), R10 + ADCQ 184(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openSSLTail192Store: - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2 - PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2 - PADDL ctr2Store, D0; PADDL ctr1Store, D1; PADDL ctr0Store, D2 + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 32(BP), X5 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 48(BP), X8 + PADDL 112(BP), X9 + PADDL 96(BP), X10 + PADDL 80(BP), X11 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X2 + PXOR X13, X5 + PXOR X14, X8 + PXOR X15, X11 + MOVOU X2, (DI) + MOVOU X5, 16(DI) + MOVOU X8, 32(DI) + MOVOU X11, 48(DI) + MOVOU 64(SI), X12 + MOVOU 80(SI), X13 + MOVOU 96(SI), X14 + MOVOU 112(SI), X15 + PXOR X12, X1 + PXOR X13, X4 + PXOR X14, X7 + PXOR X15, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + SUBQ $0x80, BX + LEAQ 128(SI), SI + LEAQ 128(DI), DI + JMP openSSETail64DecLoop - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A2; PXOR T1, B2; PXOR T2, C2; PXOR T3, D2 - MOVOU A2, (0*16)(oup); MOVOU B2, (1*16)(oup); MOVOU C2, (2*16)(oup); MOVOU D2, (3*16)(oup) - - MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3 - PXOR T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - - SUBQ $128, inl - LEAQ 128(inp), inp - LEAQ 128(oup), oup - JMP openSSETail64DecLoop - -// ---------------------------------------------------------------------------- -// Special optimization for the last 256 bytes of ciphertext openSSETail256: - // Need to decrypt up to 256 bytes - prepare four blocks - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store - XORQ itr2, itr2 + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) + XORQ R9, R9 openSSETail256Loop: - // This loop inteleaves 8 ChaCha quarter rounds with 1 poly multiplication - polyAdd(0(inp)(itr2*1)) - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - polyMulStage1 - polyMulStage2 - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyMulStage3 - polyMulReduceStage - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - ADDQ $2*8, itr2 - CMPQ itr2, $160 - JB openSSETail256Loop - MOVQ inl, itr1 - ANDQ $-16, itr1 + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + ADDQ $0x10, R9 + CMPQ R9, $0xa0 + JB openSSETail256Loop + MOVQ BX, CX + ANDQ $-16, CX openSSETail256HashLoop: - polyAdd(0(inp)(itr2*1)) - polyMul - ADDQ $2*8, itr2 - CMPQ itr2, itr1 - JB openSSETail256HashLoop + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ $0x10, R9 + CMPQ R9, CX + JB openSSETail256HashLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 - MOVO D3, tmpStore + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X6 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 80(BP), X9 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 + MOVO X15, 64(BP) // Load - xor - store - MOVOU (0*16)(inp), D3; PXOR D3, A0 - MOVOU (1*16)(inp), D3; PXOR D3, B0 - MOVOU (2*16)(inp), D3; PXOR D3, C0 - MOVOU (3*16)(inp), D3; PXOR D3, D0 - MOVOU A0, (0*16)(oup) - MOVOU B0, (1*16)(oup) - MOVOU C0, (2*16)(oup) - MOVOU D0, (3*16)(oup) - MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0 - PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0 - PXOR A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2 - MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup) - LEAQ 192(inp), inp - LEAQ 192(oup), oup - SUBQ $192, inl - MOVO A3, A0 - MOVO B3, B0 - MOVO C3, C0 - MOVO tmpStore, D0 + MOVOU (SI), X15 + PXOR X15, X0 + MOVOU 16(SI), X15 + PXOR X15, X3 + MOVOU 32(SI), X15 + PXOR X15, X6 + MOVOU 48(SI), X15 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVOU 64(SI), X0 + MOVOU 80(SI), X3 + MOVOU 96(SI), X6 + MOVOU 112(SI), X9 + PXOR X0, X1 + PXOR X3, X4 + PXOR X6, X7 + PXOR X9, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + MOVOU 128(SI), X0 + MOVOU 144(SI), X3 + MOVOU 160(SI), X6 + MOVOU 176(SI), X9 + PXOR X0, X2 + PXOR X3, X5 + PXOR X6, X8 + PXOR X9, X11 + MOVOU X2, 128(DI) + MOVOU X5, 144(DI) + MOVOU X8, 160(DI) + MOVOU X11, 176(DI) + LEAQ 192(SI), SI + LEAQ 192(DI), DI + SUBQ $0xc0, BX + MOVO X12, X0 + MOVO X13, X3 + MOVO X14, X6 + MOVO 64(BP), X9 + JMP openSSETail64DecLoop - JMP openSSETail64DecLoop - -// ---------------------------------------------------------------------------- -// ------------------------- AVX2 Code ---------------------------------------- chacha20Poly1305Open_AVX2: VZEROUPPER - VMOVDQU ·chacha20Constants<>(SB), AA0 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12 - BYTE $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4 - VPADDD ·avx2InitMask<>(SB), DD0, DD0 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x70 + BYTE $0x10 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x20 + BYTE $0xc4 + BYTE $0xc2 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x30 + VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimization, for very short buffers - CMPQ inl, $192 + CMPQ BX, $0xc0 JBE openAVX2192 - CMPQ inl, $320 + CMPQ BX, $0x00000140 JBE openAVX2320 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream - VMOVDQA BB0, state1StoreAVX2 - VMOVDQA CC0, state2StoreAVX2 - VMOVDQA DD0, ctr3StoreAVX2 - MOVQ $10, itr2 + VMOVDQA Y14, 32(BP) + VMOVDQA Y12, 64(BP) + VMOVDQA Y4, 192(BP) + MOVQ $0x0000000a, R9 openAVX2PreparePolyKey: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0 - DECQ itr2 - JNE openAVX2PreparePolyKey - - VPADDD ·chacha20Constants<>(SB), AA0, AA0 - VPADDD state1StoreAVX2, BB0, BB0 - VPADDD state2StoreAVX2, CC0, CC0 - VPADDD ctr3StoreAVX2, DD0, DD0 - - VPERM2I128 $0x02, AA0, BB0, TT0 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + DECQ R9 + JNE openAVX2PreparePolyKey + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD 32(BP), Y14, Y14 + VPADDD 64(BP), Y12, Y12 + VPADDD 192(BP), Y4, Y4 + VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for the first 64 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 // Hash AD + first 64 bytes - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) - XORQ itr1, itr1 + XORQ CX, CX openAVX2InitialHash64: - polyAdd(0(inp)(itr1*1)) - polyMulAVX2 - ADDQ $16, itr1 - CMPQ itr1, $64 - JNE openAVX2InitialHash64 + ADDQ (SI)(CX*1), R10 + ADCQ 8(SI)(CX*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ $0x10, CX + CMPQ CX, $0x40 + JNE openAVX2InitialHash64 // Decrypt the first 64 bytes - VPXOR (0*32)(inp), AA0, AA0 - VPXOR (1*32)(inp), BB0, BB0 - VMOVDQU AA0, (0*32)(oup) - VMOVDQU BB0, (1*32)(oup) - LEAQ (2*32)(inp), inp - LEAQ (2*32)(oup), oup - SUBQ $64, inl + VPXOR (SI), Y0, Y0 + VPXOR 32(SI), Y14, Y14 + VMOVDQU Y0, (DI) + VMOVDQU Y14, 32(DI) + LEAQ 64(SI), SI + LEAQ 64(DI), DI + SUBQ $0x40, BX openAVX2MainLoop: - CMPQ inl, $512 + CMPQ BX, $0x00000200 JB openAVX2MainLoopDone // Load state, increment counter blocks, store the incremented counters - VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - XORQ itr1, itr1 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + XORQ CX, CX openAVX2InternalLoop: - // Lets just say this spaghetti loop interleaves 2 quarter rounds with 3 poly multiplications - // Effectively per 512 bytes of stream we hash 480 bytes of ciphertext - polyAdd(0*8(inp)(itr1*1)) - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage1_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulStage2_AVX2 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyMulStage3_AVX2 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - polyAdd(2*8(inp)(itr1*1)) - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage1_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage2_AVX2 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage3_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulReduceStage - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(4*8(inp)(itr1*1)) - LEAQ (6*8)(itr1), itr1 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage1_AVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - polyMulStage2_AVX2 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage3_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - CMPQ itr1, $480 + ADDQ (SI)(CX*1), R10 + ADCQ 8(SI)(CX*1), R11 + ADCQ $0x01, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + ADDQ 16(SI)(CX*1), R10 + ADCQ 24(SI)(CX*1), R11 + ADCQ $0x01, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 32(SI)(CX*1), R10 + ADCQ 40(SI)(CX*1), R11 + ADCQ $0x01, R12 + LEAQ 48(CX), CX + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + CMPQ CX, $0x000001e0 JNE openAVX2InternalLoop - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) // We only hashed 480 of the 512 bytes available - hash the remaining 32 here - polyAdd(480(inp)) - polyMulAVX2 - VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0 - VPXOR (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0 - VMOVDQU CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) + ADDQ 480(SI), R10 + ADCQ 488(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) // and here - polyAdd(496(inp)) - polyMulAVX2 - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 - VPXOR (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0 - VMOVDQU AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup) - LEAQ (32*16)(inp), inp - LEAQ (32*16)(oup), oup - SUBQ $(32*16), inl + ADDQ 496(SI), R10 + ADCQ 504(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + VPXOR 384(SI), Y0, Y0 + VPXOR 416(SI), Y14, Y14 + VPXOR 448(SI), Y12, Y12 + VPXOR 480(SI), Y4, Y4 + VMOVDQU Y0, 384(DI) + VMOVDQU Y14, 416(DI) + VMOVDQU Y12, 448(DI) + VMOVDQU Y4, 480(DI) + LEAQ 512(SI), SI + LEAQ 512(DI), DI + SUBQ $0x00000200, BX JMP openAVX2MainLoop openAVX2MainLoopDone: // Handle the various tail sizes efficiently - TESTQ inl, inl + TESTQ BX, BX JE openSSEFinalize - CMPQ inl, $128 + CMPQ BX, $0x80 JBE openAVX2Tail128 - CMPQ inl, $256 + CMPQ BX, $0x00000100 JBE openAVX2Tail256 - CMPQ inl, $384 + CMPQ BX, $0x00000180 JBE openAVX2Tail384 JMP openAVX2Tail512 -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 193 bytes openAVX2192: - // For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks - VMOVDQA AA0, AA1 - VMOVDQA BB0, BB1 - VMOVDQA CC0, CC1 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2 - VMOVDQA BB0, BB2 - VMOVDQA CC0, CC2 - VMOVDQA DD0, DD2 - VMOVDQA DD1, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VMOVDQA Y4, Y2 + VMOVDQA Y1, Y15 + MOVQ $0x0000000a, R9 openAVX2192InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + DECQ R9 JNE openAVX2192InnerCipherLoop - VPADDD AA2, AA0, AA0; VPADDD AA2, AA1, AA1 - VPADDD BB2, BB0, BB0; VPADDD BB2, BB1, BB1 - VPADDD CC2, CC0, CC0; VPADDD CC2, CC1, CC1 - VPADDD DD2, DD0, DD0; VPADDD TT3, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, TT0 + VPADDD Y6, Y0, Y0 + VPADDD Y6, Y5, Y5 + VPADDD Y10, Y14, Y14 + VPADDD Y10, Y9, Y9 + VPADDD Y8, Y12, Y12 + VPADDD Y8, Y13, Y13 + VPADDD Y2, Y4, Y4 + VPADDD Y15, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 192 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 openAVX2ShortOpen: // Hash - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) openAVX2ShortOpenLoop: - CMPQ inl, $32 + CMPQ BX, $0x20 JB openAVX2ShortTail32 - SUBQ $32, inl + SUBQ $0x20, BX // Load for hashing - polyAdd(0*8(inp)) - polyMulAVX2 - polyAdd(2*8(inp)) - polyMulAVX2 + ADDQ (SI), R10 + ADCQ 8(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Load for decryption - VPXOR (inp), AA0, AA0 - VMOVDQU AA0, (oup) - LEAQ (1*32)(inp), inp - LEAQ (1*32)(oup), oup + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI + LEAQ 32(DI), DI // Shift stream left - VMOVDQA BB0, AA0 - VMOVDQA CC0, BB0 - VMOVDQA DD0, CC0 - VMOVDQA AA1, DD0 - VMOVDQA BB1, AA1 - VMOVDQA CC1, BB1 - VMOVDQA DD1, CC1 - VMOVDQA AA2, DD1 - VMOVDQA BB2, AA2 + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 + VMOVDQA Y5, Y4 + VMOVDQA Y9, Y5 + VMOVDQA Y13, Y9 + VMOVDQA Y1, Y13 + VMOVDQA Y6, Y1 + VMOVDQA Y10, Y6 JMP openAVX2ShortOpenLoop openAVX2ShortTail32: - CMPQ inl, $16 - VMOVDQA A0, A1 + CMPQ BX, $0x10 + VMOVDQA X0, X1 JB openAVX2ShortDone - - SUBQ $16, inl + SUBQ $0x10, BX // Load for hashing - polyAdd(0*8(inp)) - polyMulAVX2 + ADDQ (SI), R10 + ADCQ 8(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Load for decryption - VPXOR (inp), A0, T0 - VMOVDQU T0, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup - VPERM2I128 $0x11, AA0, AA0, AA0 - VMOVDQA A0, A1 + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 openAVX2ShortDone: VZEROUPPER JMP openSSETail16 -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 321 bytes openAVX2320: - // For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks - VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y14, Y7 + VMOVDQA Y12, Y11 + VMOVDQA Y4, Y15 + MOVQ $0x0000000a, R9 openAVX2320InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + DECQ R9 JNE openAVX2320InnerCipherLoop - - VMOVDQA ·chacha20Constants<>(SB), TT0 - VPADDD TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2 - VPADDD TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2 - VPADDD TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2 - VMOVDQA ·avx2IncMask<>(SB), TT0 - VPADDD TT3, DD0, DD0; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD1, DD1; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD2, DD2 + VMOVDQA ·chacha20Constants<>+0(SB), Y3 + VPADDD Y3, Y0, Y0 + VPADDD Y3, Y5, Y5 + VPADDD Y3, Y6, Y6 + VPADDD Y7, Y14, Y14 + VPADDD Y7, Y9, Y9 + VPADDD Y7, Y10, Y10 + VPADDD Y11, Y12, Y12 + VPADDD Y11, Y13, Y13 + VPADDD Y11, Y8, Y8 + VMOVDQA ·avx2IncMask<>+0(SB), Y3 + VPADDD Y15, Y4, Y4 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y1, Y1 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y2, Y2 // Clamp and store poly key - VPERM2I128 $0x02, AA0, BB0, TT0 - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 320 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 - VPERM2I128 $0x02, AA2, BB2, CC1 - VPERM2I128 $0x02, CC2, DD2, DD1 - VPERM2I128 $0x13, AA2, BB2, AA2 - VPERM2I128 $0x13, CC2, DD2, BB2 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 + VPERM2I128 $0x02, Y6, Y10, Y13 + VPERM2I128 $0x02, Y8, Y2, Y1 + VPERM2I128 $0x13, Y6, Y10, Y6 + VPERM2I128 $0x13, Y8, Y2, Y10 JMP openAVX2ShortOpen -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of ciphertext openAVX2Tail128: // Need to decrypt up to 128 bytes - prepare two blocks - VMOVDQA ·chacha20Constants<>(SB), AA1 - VMOVDQA state1StoreAVX2, BB1 - VMOVDQA state2StoreAVX2, CC1 - VMOVDQA ctr3StoreAVX2, DD1 - VPADDD ·avx2IncMask<>(SB), DD1, DD1 - VMOVDQA DD1, DD0 - - XORQ itr2, itr2 - MOVQ inl, itr1 - ANDQ $-16, itr1 - TESTQ itr1, itr1 - JE openAVX2Tail128LoopB + VMOVDQA ·chacha20Constants<>+0(SB), Y5 + VMOVDQA 32(BP), Y9 + VMOVDQA 64(BP), Y13 + VMOVDQA 192(BP), Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y1 + VMOVDQA Y1, Y4 + XORQ R9, R9 + MOVQ BX, CX + ANDQ $-16, CX + TESTQ CX, CX + JE openAVX2Tail128LoopB openAVX2Tail128LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMulAVX2 + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openAVX2Tail128LoopB: - ADDQ $16, itr2 - chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD1, DD1, DD1 - CMPQ itr2, itr1 - JB openAVX2Tail128LoopA - CMPQ itr2, $160 - JNE openAVX2Tail128LoopB - - VPADDD ·chacha20Constants<>(SB), AA1, AA1 - VPADDD state1StoreAVX2, BB1, BB1 - VPADDD state2StoreAVX2, CC1, CC1 - VPADDD DD0, DD1, DD1 - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 + ADDQ $0x10, R9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y1, Y1, Y1 + CMPQ R9, CX + JB openAVX2Tail128LoopA + CMPQ R9, $0xa0 + JNE openAVX2Tail128LoopB + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD 32(BP), Y9, Y9 + VPADDD 64(BP), Y13, Y13 + VPADDD Y4, Y1, Y1 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 openAVX2TailLoop: - CMPQ inl, $32 + CMPQ BX, $0x20 JB openAVX2Tail - SUBQ $32, inl + SUBQ $0x20, BX // Load for decryption - VPXOR (inp), AA0, AA0 - VMOVDQU AA0, (oup) - LEAQ (1*32)(inp), inp - LEAQ (1*32)(oup), oup - VMOVDQA BB0, AA0 - VMOVDQA CC0, BB0 - VMOVDQA DD0, CC0 + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI + LEAQ 32(DI), DI + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 JMP openAVX2TailLoop openAVX2Tail: - CMPQ inl, $16 - VMOVDQA A0, A1 + CMPQ BX, $0x10 + VMOVDQA X0, X1 JB openAVX2TailDone - SUBQ $16, inl + SUBQ $0x10, BX // Load for decryption - VPXOR (inp), A0, T0 - VMOVDQU T0, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup - VPERM2I128 $0x11, AA0, AA0, AA0 - VMOVDQA A0, A1 + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 openAVX2TailDone: VZEROUPPER JMP openSSETail16 -// ---------------------------------------------------------------------------- -// Special optimization for the last 256 bytes of ciphertext openAVX2Tail256: - // Need to decrypt up to 256 bytes - prepare four blocks - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA DD0, TT1 - VMOVDQA DD1, TT2 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y4, Y7 + VMOVDQA Y1, Y11 // Compute the number of iterations that will hash data - MOVQ inl, tmpStoreAVX2 - MOVQ inl, itr1 - SUBQ $128, itr1 - SHRQ $4, itr1 - MOVQ $10, itr2 - CMPQ itr1, $10 - CMOVQGT itr2, itr1 - MOVQ inp, inl - XORQ itr2, itr2 + MOVQ BX, 224(BP) + MOVQ BX, CX + SUBQ $0x80, CX + SHRQ $0x04, CX + MOVQ $0x0000000a, R9 + CMPQ CX, $0x0a + CMOVQGT R9, CX + MOVQ SI, BX + XORQ R9, R9 openAVX2Tail256LoopA: - polyAdd(0(inl)) - polyMulAVX2 - LEAQ 16(inl), inl + ADDQ (BX), R10 + ADCQ 8(BX), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(BX), BX - // Perform ChaCha rounds, while hashing the remaining input openAVX2Tail256LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - INCQ itr2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - CMPQ itr2, itr1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + INCQ R9 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + CMPQ R9, CX JB openAVX2Tail256LoopA + CMPQ R9, $0x0a + JNE openAVX2Tail256LoopB + MOVQ BX, R9 + SUBQ SI, BX + MOVQ BX, CX + MOVQ 224(BP), BX - CMPQ itr2, $10 - JNE openAVX2Tail256LoopB - - MOVQ inl, itr2 - SUBQ inp, inl - MOVQ inl, itr1 - MOVQ tmpStoreAVX2, inl - - // Hash the remainder of data (if any) openAVX2Tail256Hash: - ADDQ $16, itr1 - CMPQ itr1, inl - JGT openAVX2Tail256HashEnd - polyAdd (0(itr2)) - polyMulAVX2 - LEAQ 16(itr2), itr2 - JMP openAVX2Tail256Hash + ADDQ $0x10, CX + CMPQ CX, BX + JGT openAVX2Tail256HashEnd + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + JMP openAVX2Tail256Hash -// Store 128 bytes safely, then go to store loop openAVX2Tail256HashEnd: - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1 - VPADDD TT1, DD0, DD0; VPADDD TT2, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, AA2; VPERM2I128 $0x02, CC0, DD0, BB2; VPERM2I128 $0x13, AA0, BB0, CC2; VPERM2I128 $0x13, CC0, DD0, DD2 - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - - VPXOR (0*32)(inp), AA2, AA2; VPXOR (1*32)(inp), BB2, BB2; VPXOR (2*32)(inp), CC2, CC2; VPXOR (3*32)(inp), DD2, DD2 - VMOVDQU AA2, (0*32)(oup); VMOVDQU BB2, (1*32)(oup); VMOVDQU CC2, (2*32)(oup); VMOVDQU DD2, (3*32)(oup) - LEAQ (4*32)(inp), inp - LEAQ (4*32)(oup), oup - SUBQ $4*32, inl - - JMP openAVX2TailLoop - -// ---------------------------------------------------------------------------- -// Special optimization for the last 384 bytes of ciphertext -openAVX2Tail384: - // Need to decrypt up to 384 bytes - prepare six blocks - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA DD0, ctr0StoreAVX2 - VMOVDQA DD1, ctr1StoreAVX2 - VMOVDQA DD2, ctr2StoreAVX2 - - // Compute the number of iterations that will hash two blocks of data - MOVQ inl, tmpStoreAVX2 - MOVQ inl, itr1 - SUBQ $256, itr1 - SHRQ $4, itr1 - ADDQ $6, itr1 - MOVQ $10, itr2 - CMPQ itr1, $10 - CMOVQGT itr2, itr1 - MOVQ inp, inl - XORQ itr2, itr2 - - // Perform ChaCha rounds, while hashing the remaining input -openAVX2Tail384LoopB: - polyAdd(0(inl)) - polyMulAVX2 - LEAQ 16(inl), inl - -openAVX2Tail384LoopA: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - polyAdd(0(inl)) - polyMulAVX2 - LEAQ 16(inl), inl - INCQ itr2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - - CMPQ itr2, itr1 - JB openAVX2Tail384LoopB - - CMPQ itr2, $10 - JNE openAVX2Tail384LoopA - - MOVQ inl, itr2 - SUBQ inp, inl - MOVQ inl, itr1 - MOVQ tmpStoreAVX2, inl - -openAVX2Tail384Hash: - ADDQ $16, itr1 - CMPQ itr1, inl - JGT openAVX2Tail384HashEnd - polyAdd(0(itr2)) - polyMulAVX2 - LEAQ 16(itr2), itr2 - JMP openAVX2Tail384Hash - -// Store 256 bytes safely, then go to store loop -openAVX2Tail384HashEnd: - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2 - VPERM2I128 $0x02, AA0, BB0, TT0; VPERM2I128 $0x02, CC0, DD0, TT1; VPERM2I128 $0x13, AA0, BB0, TT2; VPERM2I128 $0x13, CC0, DD0, TT3 - VPXOR (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3 - VMOVDQU TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, TT0; VPERM2I128 $0x02, CC1, DD1, TT1; VPERM2I128 $0x13, AA1, BB1, TT2; VPERM2I128 $0x13, CC1, DD1, TT3 - VPXOR (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3 - VMOVDQU TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup) - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - LEAQ (8*32)(inp), inp - LEAQ (8*32)(oup), oup - SUBQ $8*32, inl + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD Y7, Y4, Y4 + VPADDD Y11, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y6 + VPERM2I128 $0x02, Y12, Y4, Y10 + VPERM2I128 $0x13, Y0, Y14, Y8 + VPERM2I128 $0x13, Y12, Y4, Y2 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR (SI), Y6, Y6 + VPXOR 32(SI), Y10, Y10 + VPXOR 64(SI), Y8, Y8 + VPXOR 96(SI), Y2, Y2 + VMOVDQU Y6, (DI) + VMOVDQU Y10, 32(DI) + VMOVDQU Y8, 64(DI) + VMOVDQU Y2, 96(DI) + LEAQ 128(SI), SI + LEAQ 128(DI), DI + SUBQ $0x80, BX + JMP openAVX2TailLoop + +openAVX2Tail384: + // Need to decrypt up to 384 bytes - prepare six blocks + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + + // Compute the number of iterations that will hash two blocks of data + MOVQ BX, 224(BP) + MOVQ BX, CX + SUBQ $0x00000100, CX + SHRQ $0x04, CX + ADDQ $0x06, CX + MOVQ $0x0000000a, R9 + CMPQ CX, $0x0a + CMOVQGT R9, CX + MOVQ SI, BX + XORQ R9, R9 + +openAVX2Tail384LoopB: + ADDQ (BX), R10 + ADCQ 8(BX), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(BX), BX + +openAVX2Tail384LoopA: + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + ADDQ (BX), R10 + ADCQ 8(BX), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(BX), BX + INCQ R9 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + CMPQ R9, CX + JB openAVX2Tail384LoopB + CMPQ R9, $0x0a + JNE openAVX2Tail384LoopA + MOVQ BX, R9 + SUBQ SI, BX + MOVQ BX, CX + MOVQ 224(BP), BX + +openAVX2Tail384Hash: + ADDQ $0x10, CX + CMPQ CX, BX + JGT openAVX2Tail384HashEnd + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + JMP openAVX2Tail384Hash + +openAVX2Tail384HashEnd: + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPERM2I128 $0x02, Y12, Y4, Y7 + VPERM2I128 $0x13, Y0, Y14, Y11 + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR (SI), Y3, Y3 + VPXOR 32(SI), Y7, Y7 + VPXOR 64(SI), Y11, Y11 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y3, (DI) + VMOVDQU Y7, 32(DI) + VMOVDQU Y11, 64(DI) + VMOVDQU Y15, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y3 + VPERM2I128 $0x02, Y13, Y1, Y7 + VPERM2I128 $0x13, Y5, Y9, Y11 + VPERM2I128 $0x13, Y13, Y1, Y15 + VPXOR 128(SI), Y3, Y3 + VPXOR 160(SI), Y7, Y7 + VPXOR 192(SI), Y11, Y11 + VPXOR 224(SI), Y15, Y15 + VMOVDQU Y3, 128(DI) + VMOVDQU Y7, 160(DI) + VMOVDQU Y11, 192(DI) + VMOVDQU Y15, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + LEAQ 256(SI), SI + LEAQ 256(DI), DI + SUBQ $0x00000100, BX JMP openAVX2TailLoop -// ---------------------------------------------------------------------------- -// Special optimization for the last 512 bytes of ciphertext openAVX2Tail512: - VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - XORQ itr1, itr1 - MOVQ inp, itr2 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + XORQ CX, CX + MOVQ SI, R9 openAVX2Tail512LoopB: - polyAdd(0(itr2)) - polyMulAVX2 - LEAQ (2*8)(itr2), itr2 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 openAVX2Tail512LoopA: - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyAdd(0*8(itr2)) - polyMulAVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(2*8(itr2)) - polyMulAVX2 - LEAQ (4*8)(itr2), itr2 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - INCQ itr1 - CMPQ itr1, $4 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 16(R9), R10 + ADCQ 24(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(R9), R9 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + INCQ CX + CMPQ CX, $0x04 JLT openAVX2Tail512LoopB - - CMPQ itr1, $10 - JNE openAVX2Tail512LoopA - - MOVQ inl, itr1 - SUBQ $384, itr1 - ANDQ $-16, itr1 + CMPQ CX, $0x0a + JNE openAVX2Tail512LoopA + MOVQ BX, CX + SUBQ $0x00000180, CX + ANDQ $-16, CX openAVX2Tail512HashLoop: - TESTQ itr1, itr1 + TESTQ CX, CX JE openAVX2Tail512HashEnd - polyAdd(0(itr2)) - polyMulAVX2 - LEAQ 16(itr2), itr2 - SUBQ $16, itr1 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + SUBQ $0x10, CX JMP openAVX2Tail512HashLoop openAVX2Tail512HashEnd: - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 - VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0 - VPXOR (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0 - VMOVDQU CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + LEAQ 384(SI), SI + LEAQ 384(DI), DI + SUBQ $0x00000180, BX + JMP openAVX2TailLoop - LEAQ (12*32)(inp), inp - LEAQ (12*32)(oup), oup - SUBQ $12*32, inl +DATA ·chacha20Constants<>+0(SB)/4, $0x61707865 +DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e +DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32 +DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574 +DATA ·chacha20Constants<>+16(SB)/4, $0x61707865 +DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e +DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 +DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 +GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 - JMP openAVX2TailLoop +DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff +DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc +DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff +DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff +GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -// func chacha20Poly1305Seal(dst, key, src, ad []byte) -TEXT ·chacha20Poly1305Seal(SB), 0, $288-96 - // For aligned stack access +DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001 +DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000 +GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16 + +DATA ·andMask<>+0(SB)/8, $0x00000000000000ff +DATA ·andMask<>+8(SB)/8, $0x0000000000000000 +DATA ·andMask<>+16(SB)/8, $0x000000000000ffff +DATA ·andMask<>+24(SB)/8, $0x0000000000000000 +DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff +DATA ·andMask<>+40(SB)/8, $0x0000000000000000 +DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff +DATA ·andMask<>+56(SB)/8, $0x0000000000000000 +DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff +DATA ·andMask<>+72(SB)/8, $0x0000000000000000 +DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff +DATA ·andMask<>+88(SB)/8, $0x0000000000000000 +DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff +DATA ·andMask<>+104(SB)/8, $0x0000000000000000 +DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+120(SB)/8, $0x0000000000000000 +DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+136(SB)/8, $0x00000000000000ff +DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+152(SB)/8, $0x000000000000ffff +DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff +DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff +DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff +DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff +DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff +GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 + +DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 +DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 + +DATA ·rol16<>+0(SB)/8, $0x0504070601000302 +DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a +DATA ·rol16<>+16(SB)/8, $0x0504070601000302 +DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a +GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 + +DATA ·rol8<>+0(SB)/8, $0x0605040702010003 +DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b +DATA ·rol8<>+16(SB)/8, $0x0605040702010003 +DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b +GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 + +DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 + +// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) +// Requires: AVX, AVX2, BMI2, CMOV, SSE2 +TEXT ·chacha20Poly1305Seal(SB), $288-96 MOVQ SP, BP - ADDQ $32, BP + ADDQ $0x20, BP ANDQ $-32, BP - MOVQ dst+0(FP), oup - MOVQ key+24(FP), keyp - MOVQ src+48(FP), inp - MOVQ src_len+56(FP), inl - MOVQ ad+72(FP), adp - - CMPB ·useAVX2(SB), $1 + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX + CMPB ·useAVX2+0(SB), $0x01 JE chacha20Poly1305Seal_AVX2 // Special optimization, for very short buffers - CMPQ inl, $128 - JBE sealSSE128 // About 15% faster + CMPQ BX, $0x80 + JBE sealSSE128 // In the seal case - prepare the poly key + 3 blocks of stream in the first iteration - MOVOU ·chacha20Constants<>(SB), A0 - MOVOU (1*16)(keyp), B0 - MOVOU (2*16)(keyp), C0 - MOVOU (3*16)(keyp), D0 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 // Store state on stack for future use - MOVO B0, state1Store - MOVO C0, state2Store + MOVO X3, 32(BP) + MOVO X6, 48(BP) // Load state, increment counter blocks - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store - MOVQ $10, itr2 + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) + MOVQ $0x0000000a, R9 sealSSEIntroLoop: - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - DECQ itr2 - JNE sealSSEIntroLoop + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + DECQ R9 + JNE sealSSEIntroLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 // Clamp and store the key - PAND ·polyClampMask<>(SB), A0 - MOVO A0, rStore - MOVO B0, sStore + PAND ·polyClampMask<>+0(SB), X0 + MOVO X0, (BP) + MOVO X3, 16(BP) // Hash AAD - MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal<>(SB) - - MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0 - PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1 - MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup) - MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0 - PXOR A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2 - MOVOU A2, (4*16)(oup); MOVOU B2, (5*16)(oup); MOVOU C2, (6*16)(oup); MOVOU D2, (7*16)(oup) - - MOVQ $128, itr1 - SUBQ $128, inl - LEAQ 128(inp), inp - - MOVO A3, A1; MOVO B3, B1; MOVO C3, C1; MOVO D3, D1 - - CMPQ inl, $64 - JBE sealSSE128SealHash - - MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0 - PXOR A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3 - MOVOU A3, (8*16)(oup); MOVOU B3, (9*16)(oup); MOVOU C3, (10*16)(oup); MOVOU D3, (11*16)(oup) - - ADDQ $64, itr1 - SUBQ $64, inl - LEAQ 64(inp), inp - - MOVQ $2, itr1 - MOVQ $8, itr2 - - CMPQ inl, $64 - JBE sealSSETail64 - CMPQ inl, $128 - JBE sealSSETail128 - CMPQ inl, $192 - JBE sealSSETail192 + MOVQ ad_len+80(FP), R9 + CALL polyHashADInternal<>(SB) + MOVOU (SI), X0 + MOVOU 16(SI), X3 + MOVOU 32(SI), X6 + MOVOU 48(SI), X9 + PXOR X0, X1 + PXOR X3, X4 + PXOR X6, X7 + PXOR X9, X10 + MOVOU X1, (DI) + MOVOU X4, 16(DI) + MOVOU X7, 32(DI) + MOVOU X10, 48(DI) + MOVOU 64(SI), X0 + MOVOU 80(SI), X3 + MOVOU 96(SI), X6 + MOVOU 112(SI), X9 + PXOR X0, X2 + PXOR X3, X5 + PXOR X6, X8 + PXOR X9, X11 + MOVOU X2, 64(DI) + MOVOU X5, 80(DI) + MOVOU X8, 96(DI) + MOVOU X11, 112(DI) + MOVQ $0x00000080, CX + SUBQ $0x80, BX + LEAQ 128(SI), SI + MOVO X12, X1 + MOVO X13, X4 + MOVO X14, X7 + MOVO X15, X10 + CMPQ BX, $0x40 + JBE sealSSE128SealHash + MOVOU (SI), X0 + MOVOU 16(SI), X3 + MOVOU 32(SI), X6 + MOVOU 48(SI), X9 + PXOR X0, X12 + PXOR X3, X13 + PXOR X6, X14 + PXOR X9, X15 + MOVOU X12, 128(DI) + MOVOU X13, 144(DI) + MOVOU X14, 160(DI) + MOVOU X15, 176(DI) + ADDQ $0x40, CX + SUBQ $0x40, BX + LEAQ 64(SI), SI + MOVQ $0x00000002, CX + MOVQ $0x00000008, R9 + CMPQ BX, $0x40 + JBE sealSSETail64 + CMPQ BX, $0x80 + JBE sealSSETail128 + CMPQ BX, $0xc0 + JBE sealSSETail192 sealSSEMainLoop: // Load state, increment counter blocks - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) sealSSEInnerLoop: - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyAdd(0(oup)) - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - polyMulStage1 - polyMulStage2 - LEAQ (2*8)(oup), oup - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - polyMulStage3 - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyMulReduceStage - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - DECQ itr2 - JGE sealSSEInnerLoop - polyAdd(0(oup)) - polyMul - LEAQ (2*8)(oup), oup - DECQ itr1 - JG sealSSEInnerLoop + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + LEAQ 16(DI), DI + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + DECQ R9 + JGE sealSSEInnerLoop + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + DECQ CX + JG sealSSEInnerLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 - MOVO D3, tmpStore + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X6 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 80(BP), X9 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 + MOVO X15, 64(BP) // Load - xor - store - MOVOU (0*16)(inp), D3; PXOR D3, A0 - MOVOU (1*16)(inp), D3; PXOR D3, B0 - MOVOU (2*16)(inp), D3; PXOR D3, C0 - MOVOU (3*16)(inp), D3; PXOR D3, D0 - MOVOU A0, (0*16)(oup) - MOVOU B0, (1*16)(oup) - MOVOU C0, (2*16)(oup) - MOVOU D0, (3*16)(oup) - MOVO tmpStore, D3 - - MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0 - PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0 - PXOR A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2 - MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup) - ADDQ $192, inp - MOVQ $192, itr1 - SUBQ $192, inl - MOVO A3, A1 - MOVO B3, B1 - MOVO C3, C1 - MOVO D3, D1 - CMPQ inl, $64 + MOVOU (SI), X15 + PXOR X15, X0 + MOVOU 16(SI), X15 + PXOR X15, X3 + MOVOU 32(SI), X15 + PXOR X15, X6 + MOVOU 48(SI), X15 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVO 64(BP), X15 + MOVOU 64(SI), X0 + MOVOU 80(SI), X3 + MOVOU 96(SI), X6 + MOVOU 112(SI), X9 + PXOR X0, X1 + PXOR X3, X4 + PXOR X6, X7 + PXOR X9, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + MOVOU 128(SI), X0 + MOVOU 144(SI), X3 + MOVOU 160(SI), X6 + MOVOU 176(SI), X9 + PXOR X0, X2 + PXOR X3, X5 + PXOR X6, X8 + PXOR X9, X11 + MOVOU X2, 128(DI) + MOVOU X5, 144(DI) + MOVOU X8, 160(DI) + MOVOU X11, 176(DI) + ADDQ $0xc0, SI + MOVQ $0x000000c0, CX + SUBQ $0xc0, BX + MOVO X12, X1 + MOVO X13, X4 + MOVO X14, X7 + MOVO X15, X10 + CMPQ BX, $0x40 JBE sealSSE128SealHash - MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0 - PXOR A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3 - MOVOU A3, (12*16)(oup); MOVOU B3, (13*16)(oup); MOVOU C3, (14*16)(oup); MOVOU D3, (15*16)(oup) - LEAQ 64(inp), inp - SUBQ $64, inl - MOVQ $6, itr1 - MOVQ $4, itr2 - CMPQ inl, $192 + MOVOU (SI), X0 + MOVOU 16(SI), X3 + MOVOU 32(SI), X6 + MOVOU 48(SI), X9 + PXOR X0, X12 + PXOR X3, X13 + PXOR X6, X14 + PXOR X9, X15 + MOVOU X12, 192(DI) + MOVOU X13, 208(DI) + MOVOU X14, 224(DI) + MOVOU X15, 240(DI) + LEAQ 64(SI), SI + SUBQ $0x40, BX + MOVQ $0x00000006, CX + MOVQ $0x00000004, R9 + CMPQ BX, $0xc0 JG sealSSEMainLoop - - MOVQ inl, itr1 - TESTQ inl, inl + MOVQ BX, CX + TESTQ BX, BX JE sealSSE128SealHash - MOVQ $6, itr1 - CMPQ inl, $64 + MOVQ $0x00000006, CX + CMPQ BX, $0x40 JBE sealSSETail64 - CMPQ inl, $128 + CMPQ BX, $0x80 JBE sealSSETail128 JMP sealSSETail192 -// ---------------------------------------------------------------------------- -// Special optimization for the last 64 bytes of plaintext sealSSETail64: - // Need to encrypt up to 64 bytes - prepare single block, hash 192 or 256 bytes - MOVO ·chacha20Constants<>(SB), A1 - MOVO state1Store, B1 - MOVO state2Store, C1 - MOVO ctr3Store, D1 - PADDL ·sseIncMask<>(SB), D1 - MOVO D1, ctr0Store + MOVO ·chacha20Constants<>+0(SB), X1 + MOVO 32(BP), X4 + MOVO 48(BP), X7 + MOVO 128(BP), X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 80(BP) sealSSETail64LoopA: - // Perform ChaCha rounds, while hashing the previously encrypted ciphertext - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealSSETail64LoopB: - chachaQR(A1, B1, C1, D1, T1) - shiftB1Left; shiftC1Left; shiftD1Left - chachaQR(A1, B1, C1, D1, T1) - shiftB1Right; shiftC1Right; shiftD1Right - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup - - DECQ itr1 - JG sealSSETail64LoopA - - DECQ itr2 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x0c, X13 + PSRLL $0x14, X4 + PXOR X13, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x07, X13 + PSRLL $0x19, X4 + PXOR X13, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x0c, X13 + PSRLL $0x14, X4 + PXOR X13, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x07, X13 + PSRLL $0x19, X4 + PXOR X13, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + DECQ CX + JG sealSSETail64LoopA + DECQ R9 JGE sealSSETail64LoopB - PADDL ·chacha20Constants<>(SB), A1 - PADDL state1Store, B1 - PADDL state2Store, C1 - PADDL ctr0Store, D1 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL 32(BP), X4 + PADDL 48(BP), X7 + PADDL 80(BP), X10 + JMP sealSSE128Seal - JMP sealSSE128Seal - -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of plaintext sealSSETail128: - // Need to encrypt up to 128 bytes - prepare two blocks, hash 192 or 256 bytes - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 80(BP) + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 96(BP) sealSSETail128LoopA: - // Perform ChaCha rounds, while hashing the previously encrypted ciphertext - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealSSETail128LoopB: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + DECQ CX + JG sealSSETail128LoopA + DECQ R9 + JGE sealSSETail128LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 80(BP), X9 + PADDL 96(BP), X10 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X0 + PXOR X13, X3 + PXOR X14, X6 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVQ $0x00000040, CX + LEAQ 64(SI), SI + SUBQ $0x40, BX + JMP sealSSE128SealHash - DECQ itr1 - JG sealSSETail128LoopA - - DECQ itr2 - JGE sealSSETail128LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1 - PADDL state1Store, B0; PADDL state1Store, B1 - PADDL state2Store, C0; PADDL state2Store, C1 - PADDL ctr0Store, D0; PADDL ctr1Store, D1 - - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0 - MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup) - - MOVQ $64, itr1 - LEAQ 64(inp), inp - SUBQ $64, inl - - JMP sealSSE128SealHash - -// ---------------------------------------------------------------------------- -// Special optimization for the last 192 bytes of plaintext sealSSETail192: - // Need to encrypt up to 192 bytes - prepare three blocks, hash 192 or 256 bytes - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr2Store + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 80(BP) + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 96(BP) + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X11, 112(BP) sealSSETail192LoopA: - // Perform ChaCha rounds, while hashing the previously encrypted ciphertext - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealSSETail192LoopB: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - shiftB2Left; shiftC2Left; shiftD2Left + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + DECQ CX + JG sealSSETail192LoopA + DECQ R9 + JGE sealSSETail192LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 32(BP), X5 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 48(BP), X8 + PADDL 80(BP), X9 + PADDL 96(BP), X10 + PADDL 112(BP), X11 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X0 + PXOR X13, X3 + PXOR X14, X6 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVOU 64(SI), X12 + MOVOU 80(SI), X13 + MOVOU 96(SI), X14 + MOVOU 112(SI), X15 + PXOR X12, X1 + PXOR X13, X4 + PXOR X14, X7 + PXOR X15, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + MOVO X2, X1 + MOVO X5, X4 + MOVO X8, X7 + MOVO X11, X10 + MOVQ $0x00000080, CX + LEAQ 128(SI), SI + SUBQ $0x80, BX + JMP sealSSE128SealHash - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup - - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right - shiftB2Right; shiftC2Right; shiftD2Right - - DECQ itr1 - JG sealSSETail192LoopA - - DECQ itr2 - JGE sealSSETail192LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2 - PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2 - PADDL ctr0Store, D0; PADDL ctr1Store, D1; PADDL ctr2Store, D2 - - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0 - MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup) - MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3 - PXOR T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - - MOVO A2, A1 - MOVO B2, B1 - MOVO C2, C1 - MOVO D2, D1 - MOVQ $128, itr1 - LEAQ 128(inp), inp - SUBQ $128, inl - - JMP sealSSE128SealHash - -// ---------------------------------------------------------------------------- -// Special seal optimization for buffers smaller than 129 bytes sealSSE128: - // For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks - MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO B0, T1; MOVO C0, T2; MOVO D1, T3 - MOVQ $10, itr2 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X3, X13 + MOVO X6, X14 + MOVO X10, X15 + MOVQ $0x0000000a, R9 sealSSE128InnerCipherLoop: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftB1Left; shiftB2Left - shiftC0Left; shiftC1Left; shiftC2Left - shiftD0Left; shiftD1Left; shiftD2Left - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftB1Right; shiftB2Right - shiftC0Right; shiftC1Right; shiftC2Right - shiftD0Right; shiftD1Right; shiftD2Right - DECQ itr2 - JNE sealSSE128InnerCipherLoop + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + DECQ R9 + JNE sealSSE128InnerCipherLoop // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL T1, B0; PADDL T1, B1; PADDL T1, B2 - PADDL T2, C1; PADDL T2, C2 - PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2 - PAND ·polyClampMask<>(SB), A0 - MOVOU A0, rStore - MOVOU B0, sStore + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL X13, X3 + PADDL X13, X4 + PADDL X13, X5 + PADDL X14, X7 + PADDL X14, X8 + PADDL X15, X10 + PADDL ·sseIncMask<>+0(SB), X15 + PADDL X15, X11 + PAND ·polyClampMask<>+0(SB), X0 + MOVOU X0, (BP) + MOVOU X3, 16(BP) // Hash - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) - XORQ itr1, itr1 + XORQ CX, CX sealSSE128SealHash: - // itr1 holds the number of bytes encrypted but not yet hashed - CMPQ itr1, $16 - JB sealSSE128Seal - polyAdd(0(oup)) - polyMul - - SUBQ $16, itr1 - ADDQ $16, oup - - JMP sealSSE128SealHash + CMPQ CX, $0x10 + JB sealSSE128Seal + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + SUBQ $0x10, CX + ADDQ $0x10, DI + JMP sealSSE128SealHash sealSSE128Seal: - CMPQ inl, $16 + CMPQ BX, $0x10 JB sealSSETail - SUBQ $16, inl + SUBQ $0x10, BX // Load for decryption - MOVOU (inp), T0 - PXOR T0, A1 - MOVOU A1, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup + MOVOU (SI), X12 + PXOR X12, X1 + MOVOU X1, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI // Extract for hashing - MOVQ A1, t0 - PSRLDQ $8, A1 - MOVQ A1, t1 - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul + MOVQ X1, R13 + PSRLDQ $0x08, X1 + MOVQ X1, R14 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Shift the stream "left" - MOVO B1, A1 - MOVO C1, B1 - MOVO D1, C1 - MOVO A2, D1 - MOVO B2, A2 - MOVO C2, B2 - MOVO D2, C2 + MOVO X4, X1 + MOVO X7, X4 + MOVO X10, X7 + MOVO X2, X10 + MOVO X5, X2 + MOVO X8, X5 + MOVO X11, X8 JMP sealSSE128Seal sealSSETail: - TESTQ inl, inl + TESTQ BX, BX JE sealSSEFinalize // We can only load the PT one byte at a time to avoid read after end of buffer - MOVQ inl, itr2 - SHLQ $4, itr2 - LEAQ ·andMask<>(SB), t0 - MOVQ inl, itr1 - LEAQ -1(inp)(inl*1), inp - XORQ t2, t2 - XORQ t3, t3 + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVQ BX, CX + LEAQ -1(SI)(BX*1), SI + XORQ R15, R15 + XORQ R8, R8 XORQ AX, AX sealSSETailLoadLoop: - SHLQ $8, t2, t3 - SHLQ $8, t2 - MOVB (inp), AX - XORQ AX, t2 - LEAQ -1(inp), inp - DECQ itr1 + SHLQ $0x08, R15, R8 + SHLQ $0x08, R15 + MOVB (SI), AX + XORQ AX, R15 + LEAQ -1(SI), SI + DECQ CX JNE sealSSETailLoadLoop - MOVQ t2, 0+tmpStore - MOVQ t3, 8+tmpStore - PXOR 0+tmpStore, A1 - MOVOU A1, (oup) - MOVOU -16(t0)(itr2*1), T0 - PAND T0, A1 - MOVQ A1, t0 - PSRLDQ $8, A1 - MOVQ A1, t1 - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul - - ADDQ inl, oup + MOVQ R15, 64(BP) + MOVQ R8, 72(BP) + PXOR 64(BP), X1 + MOVOU X1, (DI) + MOVOU -16(R13)(R9*1), X12 + PAND X12, X1 + MOVQ X1, R13 + PSRLDQ $0x08, X1 + MOVQ X1, R14 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ BX, DI sealSSEFinalize: // Hash in the buffer lengths - ADDQ ad_len+80(FP), acc0 - ADCQ src_len+56(FP), acc1 - ADCQ $1, acc2 - polyMul + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Final reduce - MOVQ acc0, t0 - MOVQ acc1, t1 - MOVQ acc2, t2 - SUBQ $-5, acc0 - SBBQ $-1, acc1 - SBBQ $3, acc2 - CMOVQCS t0, acc0 - CMOVQCS t1, acc1 - CMOVQCS t2, acc2 + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 // Add in the "s" part of the key - ADDQ 0+sStore, acc0 - ADCQ 8+sStore, acc1 + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 // Finally store the tag at the end of the message - MOVQ acc0, (0*8)(oup) - MOVQ acc1, (1*8)(oup) + MOVQ R10, (DI) + MOVQ R11, 8(DI) RET -// ---------------------------------------------------------------------------- -// ------------------------- AVX2 Code ---------------------------------------- chacha20Poly1305Seal_AVX2: VZEROUPPER - VMOVDQU ·chacha20Constants<>(SB), AA0 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12 - BYTE $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4 - VPADDD ·avx2InitMask<>(SB), DD0, DD0 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x70 + BYTE $0x10 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x20 + BYTE $0xc4 + BYTE $0xc2 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x30 + VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimizations, for very short buffers - CMPQ inl, $192 - JBE seal192AVX2 // 33% faster - CMPQ inl, $320 - JBE seal320AVX2 // 17% faster + CMPQ BX, $0x000000c0 + JBE seal192AVX2 + CMPQ BX, $0x00000140 + JBE seal320AVX2 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream - VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3; VMOVDQA BB0, state1StoreAVX2 - VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3; VMOVDQA CC0, state2StoreAVX2 - VPADDD ·avx2IncMask<>(SB), DD0, DD1; VMOVDQA DD0, ctr0StoreAVX2 - VPADDD ·avx2IncMask<>(SB), DD1, DD2; VMOVDQA DD1, ctr1StoreAVX2 - VPADDD ·avx2IncMask<>(SB), DD2, DD3; VMOVDQA DD2, ctr2StoreAVX2 - VMOVDQA DD3, ctr3StoreAVX2 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA Y14, 32(BP) + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA Y12, 64(BP) + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y4, 96(BP) + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y1, 128(BP) + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + MOVQ $0x0000000a, R9 sealAVX2IntroLoop: - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0 - VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1 - VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2 - VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3 - - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0 - VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1 - VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2 - VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3 - DECQ itr2 - JNE sealAVX2IntroLoop - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - - VPERM2I128 $0x13, CC0, DD0, CC0 // Stream bytes 96 - 127 - VPERM2I128 $0x02, AA0, BB0, DD0 // The Poly1305 key - VPERM2I128 $0x13, AA0, BB0, AA0 // Stream bytes 64 - 95 + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y3, Y3, Y3 + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y3, Y3, Y3 + DECQ R9 + JNE sealAVX2IntroLoop + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPERM2I128 $0x02, Y0, Y14, Y4 + VPERM2I128 $0x13, Y0, Y14, Y0 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), DD0, DD0 - VMOVDQA DD0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y4, Y4 + VMOVDQA Y4, (BP) // Hash AD - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) // Can store at least 320 bytes - VPXOR (0*32)(inp), AA0, AA0 - VPXOR (1*32)(inp), CC0, CC0 - VMOVDQU AA0, (0*32)(oup) - VMOVDQU CC0, (1*32)(oup) - - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (2*32)(inp), AA0, AA0; VPXOR (3*32)(inp), BB0, BB0; VPXOR (4*32)(inp), CC0, CC0; VPXOR (5*32)(inp), DD0, DD0 - VMOVDQU AA0, (2*32)(oup); VMOVDQU BB0, (3*32)(oup); VMOVDQU CC0, (4*32)(oup); VMOVDQU DD0, (5*32)(oup) - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (6*32)(inp), AA0, AA0; VPXOR (7*32)(inp), BB0, BB0; VPXOR (8*32)(inp), CC0, CC0; VPXOR (9*32)(inp), DD0, DD0 - VMOVDQU AA0, (6*32)(oup); VMOVDQU BB0, (7*32)(oup); VMOVDQU CC0, (8*32)(oup); VMOVDQU DD0, (9*32)(oup) - - MOVQ $320, itr1 - SUBQ $320, inl - LEAQ 320(inp), inp - - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, CC3, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, CC3, DD3, DD0 - CMPQ inl, $128 + VPXOR (SI), Y0, Y0 + VPXOR 32(SI), Y12, Y12 + VMOVDQU Y0, (DI) + VMOVDQU Y12, 32(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 64(SI), Y0, Y0 + VPXOR 96(SI), Y14, Y14 + VPXOR 128(SI), Y12, Y12 + VPXOR 160(SI), Y4, Y4 + VMOVDQU Y0, 64(DI) + VMOVDQU Y14, 96(DI) + VMOVDQU Y12, 128(DI) + VMOVDQU Y4, 160(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 192(SI), Y0, Y0 + VPXOR 224(SI), Y14, Y14 + VPXOR 256(SI), Y12, Y12 + VPXOR 288(SI), Y4, Y4 + VMOVDQU Y0, 192(DI) + VMOVDQU Y14, 224(DI) + VMOVDQU Y12, 256(DI) + VMOVDQU Y4, 288(DI) + MOVQ $0x00000140, CX + SUBQ $0x00000140, BX + LEAQ 320(SI), SI + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, Y15, Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, Y15, Y3, Y4 + CMPQ BX, $0x80 JBE sealAVX2SealHash - - VPXOR (0*32)(inp), AA0, AA0; VPXOR (1*32)(inp), BB0, BB0; VPXOR (2*32)(inp), CC0, CC0; VPXOR (3*32)(inp), DD0, DD0 - VMOVDQU AA0, (10*32)(oup); VMOVDQU BB0, (11*32)(oup); VMOVDQU CC0, (12*32)(oup); VMOVDQU DD0, (13*32)(oup) - SUBQ $128, inl - LEAQ 128(inp), inp - - MOVQ $8, itr1 - MOVQ $2, itr2 - - CMPQ inl, $128 - JBE sealAVX2Tail128 - CMPQ inl, $256 - JBE sealAVX2Tail256 - CMPQ inl, $384 - JBE sealAVX2Tail384 - CMPQ inl, $512 - JBE sealAVX2Tail512 + VPXOR (SI), Y0, Y0 + VPXOR 32(SI), Y14, Y14 + VPXOR 64(SI), Y12, Y12 + VPXOR 96(SI), Y4, Y4 + VMOVDQU Y0, 320(DI) + VMOVDQU Y14, 352(DI) + VMOVDQU Y12, 384(DI) + VMOVDQU Y4, 416(DI) + SUBQ $0x80, BX + LEAQ 128(SI), SI + MOVQ $0x00000008, CX + MOVQ $0x00000002, R9 + CMPQ BX, $0x80 + JBE sealAVX2Tail128 + CMPQ BX, $0x00000100 + JBE sealAVX2Tail256 + CMPQ BX, $0x00000180 + JBE sealAVX2Tail384 + CMPQ BX, $0x00000200 + JBE sealAVX2Tail512 // We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0 - VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1 - VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2 - VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3 - - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0 - VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1 - VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2 - VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - - SUBQ $16, oup // Adjust the pointer - MOVQ $9, itr1 - JMP sealAVX2InternalLoopStart + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y3, Y3, Y3 + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + SUBQ $0x10, DI + MOVQ $0x00000009, CX + JMP sealAVX2InternalLoopStart sealAVX2MainLoop: - // Load state, increment counter blocks, store the incremented counters - VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - MOVQ $10, itr1 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + MOVQ $0x0000000a, CX sealAVX2InternalLoop: - polyAdd(0*8(oup)) - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage1_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulStage2_AVX2 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyMulStage3_AVX2 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 sealAVX2InternalLoopStart: - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - polyAdd(2*8(oup)) - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage1_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage2_AVX2 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage3_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulReduceStage - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(4*8(oup)) - LEAQ (6*8)(oup), oup - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage1_AVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - polyMulStage2_AVX2 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage3_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - DECQ itr1 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 32(DI), R10 + ADCQ 40(DI), R11 + ADCQ $0x01, R12 + LEAQ 48(DI), DI + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + DECQ CX JNE sealAVX2InternalLoop - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) // We only hashed 480 of the 512 bytes available - hash the remaining 32 here - polyAdd(0*8(oup)) - polyMulAVX2 - LEAQ (4*8)(oup), oup - VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0 - VPXOR (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0 - VMOVDQU CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) // and here - polyAdd(-2*8(oup)) - polyMulAVX2 - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 - VPXOR (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0 - VMOVDQU AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup) - LEAQ (32*16)(inp), inp - SUBQ $(32*16), inl - CMPQ inl, $512 + ADDQ -16(DI), R10 + ADCQ -8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + VPXOR 384(SI), Y0, Y0 + VPXOR 416(SI), Y14, Y14 + VPXOR 448(SI), Y12, Y12 + VPXOR 480(SI), Y4, Y4 + VMOVDQU Y0, 384(DI) + VMOVDQU Y14, 416(DI) + VMOVDQU Y12, 448(DI) + VMOVDQU Y4, 480(DI) + LEAQ 512(SI), SI + SUBQ $0x00000200, BX + CMPQ BX, $0x00000200 JG sealAVX2MainLoop // Tail can only hash 480 bytes - polyAdd(0*8(oup)) - polyMulAVX2 - polyAdd(2*8(oup)) - polyMulAVX2 - LEAQ 32(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + MOVQ $0x0000000a, CX + MOVQ $0x00000000, R9 + CMPQ BX, $0x80 + JBE sealAVX2Tail128 + CMPQ BX, $0x00000100 + JBE sealAVX2Tail256 + CMPQ BX, $0x00000180 + JBE sealAVX2Tail384 + JMP sealAVX2Tail512 - MOVQ $10, itr1 - MOVQ $0, itr2 - CMPQ inl, $128 - JBE sealAVX2Tail128 - CMPQ inl, $256 - JBE sealAVX2Tail256 - CMPQ inl, $384 - JBE sealAVX2Tail384 - JMP sealAVX2Tail512 - -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 193 bytes seal192AVX2: - // For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks - VMOVDQA AA0, AA1 - VMOVDQA BB0, BB1 - VMOVDQA CC0, CC1 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2 - VMOVDQA BB0, BB2 - VMOVDQA CC0, CC2 - VMOVDQA DD0, DD2 - VMOVDQA DD1, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VMOVDQA Y4, Y2 + VMOVDQA Y1, Y15 + MOVQ $0x0000000a, R9 sealAVX2192InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + DECQ R9 JNE sealAVX2192InnerCipherLoop - VPADDD AA2, AA0, AA0; VPADDD AA2, AA1, AA1 - VPADDD BB2, BB0, BB0; VPADDD BB2, BB1, BB1 - VPADDD CC2, CC0, CC0; VPADDD CC2, CC1, CC1 - VPADDD DD2, DD0, DD0; VPADDD TT3, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, TT0 + VPADDD Y6, Y0, Y0 + VPADDD Y6, Y5, Y5 + VPADDD Y10, Y14, Y14 + VPADDD Y10, Y9, Y9 + VPADDD Y8, Y12, Y12 + VPADDD Y8, Y13, Y13 + VPADDD Y2, Y4, Y4 + VPADDD Y15, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 192 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 sealAVX2ShortSeal: // Hash aad - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) - XORQ itr1, itr1 + XORQ CX, CX sealAVX2SealHash: // itr1 holds the number of bytes encrypted but not yet hashed - CMPQ itr1, $16 - JB sealAVX2ShortSealLoop - polyAdd(0(oup)) - polyMul - SUBQ $16, itr1 - ADDQ $16, oup - JMP sealAVX2SealHash + CMPQ CX, $0x10 + JB sealAVX2ShortSealLoop + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + SUBQ $0x10, CX + ADDQ $0x10, DI + JMP sealAVX2SealHash sealAVX2ShortSealLoop: - CMPQ inl, $32 + CMPQ BX, $0x20 JB sealAVX2ShortTail32 - SUBQ $32, inl + SUBQ $0x20, BX // Load for encryption - VPXOR (inp), AA0, AA0 - VMOVDQU AA0, (oup) - LEAQ (1*32)(inp), inp + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI // Now can hash - polyAdd(0*8(oup)) - polyMulAVX2 - polyAdd(2*8(oup)) - polyMulAVX2 - LEAQ (1*32)(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI // Shift stream left - VMOVDQA BB0, AA0 - VMOVDQA CC0, BB0 - VMOVDQA DD0, CC0 - VMOVDQA AA1, DD0 - VMOVDQA BB1, AA1 - VMOVDQA CC1, BB1 - VMOVDQA DD1, CC1 - VMOVDQA AA2, DD1 - VMOVDQA BB2, AA2 + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 + VMOVDQA Y5, Y4 + VMOVDQA Y9, Y5 + VMOVDQA Y13, Y9 + VMOVDQA Y1, Y13 + VMOVDQA Y6, Y1 + VMOVDQA Y10, Y6 JMP sealAVX2ShortSealLoop sealAVX2ShortTail32: - CMPQ inl, $16 - VMOVDQA A0, A1 + CMPQ BX, $0x10 + VMOVDQA X0, X1 JB sealAVX2ShortDone - - SUBQ $16, inl + SUBQ $0x10, BX // Load for encryption - VPXOR (inp), A0, T0 - VMOVDQU T0, (oup) - LEAQ (1*16)(inp), inp + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI // Hash - polyAdd(0*8(oup)) - polyMulAVX2 - LEAQ (1*16)(oup), oup - VPERM2I128 $0x11, AA0, AA0, AA0 - VMOVDQA A0, A1 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 sealAVX2ShortDone: VZEROUPPER JMP sealSSETail -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 321 bytes seal320AVX2: - // For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks - VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y14, Y7 + VMOVDQA Y12, Y11 + VMOVDQA Y4, Y15 + MOVQ $0x0000000a, R9 sealAVX2320InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + DECQ R9 JNE sealAVX2320InnerCipherLoop - - VMOVDQA ·chacha20Constants<>(SB), TT0 - VPADDD TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2 - VPADDD TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2 - VPADDD TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2 - VMOVDQA ·avx2IncMask<>(SB), TT0 - VPADDD TT3, DD0, DD0; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD1, DD1; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD2, DD2 + VMOVDQA ·chacha20Constants<>+0(SB), Y3 + VPADDD Y3, Y0, Y0 + VPADDD Y3, Y5, Y5 + VPADDD Y3, Y6, Y6 + VPADDD Y7, Y14, Y14 + VPADDD Y7, Y9, Y9 + VPADDD Y7, Y10, Y10 + VPADDD Y11, Y12, Y12 + VPADDD Y11, Y13, Y13 + VPADDD Y11, Y8, Y8 + VMOVDQA ·avx2IncMask<>+0(SB), Y3 + VPADDD Y15, Y4, Y4 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y1, Y1 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y2, Y2 // Clamp and store poly key - VPERM2I128 $0x02, AA0, BB0, TT0 - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 320 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 - VPERM2I128 $0x02, AA2, BB2, CC1 - VPERM2I128 $0x02, CC2, DD2, DD1 - VPERM2I128 $0x13, AA2, BB2, AA2 - VPERM2I128 $0x13, CC2, DD2, BB2 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 + VPERM2I128 $0x02, Y6, Y10, Y13 + VPERM2I128 $0x02, Y8, Y2, Y1 + VPERM2I128 $0x13, Y6, Y10, Y6 + VPERM2I128 $0x13, Y8, Y2, Y10 JMP sealAVX2ShortSeal -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of ciphertext sealAVX2Tail128: - // Need to decrypt up to 128 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0 - VMOVDQA state1StoreAVX2, BB0 - VMOVDQA state2StoreAVX2, CC0 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VMOVDQA DD0, DD1 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA 32(BP), Y14 + VMOVDQA 64(BP), Y12 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VMOVDQA Y4, Y1 sealAVX2Tail128LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail128LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - polyAdd(0(oup)) - polyMul - VPALIGNR $4, BB0, BB0, BB0 - VPALIGNR $8, CC0, CC0, CC0 - VPALIGNR $12, DD0, DD0, DD0 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - polyAdd(16(oup)) - polyMul - LEAQ 32(oup), oup - VPALIGNR $12, BB0, BB0, BB0 - VPALIGNR $8, CC0, CC0, CC0 - VPALIGNR $4, DD0, DD0, DD0 - DECQ itr1 - JG sealAVX2Tail128LoopA - DECQ itr2 - JGE sealAVX2Tail128LoopB - - VPADDD ·chacha20Constants<>(SB), AA0, AA1 - VPADDD state1StoreAVX2, BB0, BB1 - VPADDD state2StoreAVX2, CC0, CC1 - VPADDD DD1, DD0, DD1 - - VPERM2I128 $0x02, AA1, BB1, AA0 - VPERM2I128 $0x02, CC1, DD1, BB0 - VPERM2I128 $0x13, AA1, BB1, CC0 - VPERM2I128 $0x13, CC1, DD1, DD0 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + DECQ CX + JG sealAVX2Tail128LoopA + DECQ R9 + JGE sealAVX2Tail128LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y5 + VPADDD 32(BP), Y14, Y9 + VPADDD 64(BP), Y12, Y13 + VPADDD Y1, Y4, Y1 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 JMP sealAVX2ShortSealLoop -// ---------------------------------------------------------------------------- -// Special optimization for the last 256 bytes of ciphertext sealAVX2Tail256: - // Need to decrypt up to 256 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA ·chacha20Constants<>(SB), AA1 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA state1StoreAVX2, BB1 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA state2StoreAVX2, CC1 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA DD0, TT1 - VMOVDQA DD1, TT2 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA ·chacha20Constants<>+0(SB), Y5 + VMOVDQA 32(BP), Y14 + VMOVDQA 32(BP), Y9 + VMOVDQA 64(BP), Y12 + VMOVDQA 64(BP), Y13 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y4, Y7 + VMOVDQA Y1, Y11 sealAVX2Tail256LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail256LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - polyAdd(0(oup)) - polyMul - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - polyAdd(16(oup)) - polyMul - LEAQ 32(oup), oup - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - DECQ itr1 - JG sealAVX2Tail256LoopA - DECQ itr2 - JGE sealAVX2Tail256LoopB + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + DECQ CX + JG sealAVX2Tail256LoopA + DECQ R9 + JGE sealAVX2Tail256LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD Y7, Y4, Y4 + VPADDD Y11, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPERM2I128 $0x02, Y12, Y4, Y7 + VPERM2I128 $0x13, Y0, Y14, Y11 + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR (SI), Y3, Y3 + VPXOR 32(SI), Y7, Y7 + VPXOR 64(SI), Y11, Y11 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y3, (DI) + VMOVDQU Y7, 32(DI) + VMOVDQU Y11, 64(DI) + VMOVDQU Y15, 96(DI) + MOVQ $0x00000080, CX + LEAQ 128(SI), SI + SUBQ $0x80, BX + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + JMP sealAVX2SealHash - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1 - VPADDD TT1, DD0, DD0; VPADDD TT2, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, TT0 - VPERM2I128 $0x02, CC0, DD0, TT1 - VPERM2I128 $0x13, AA0, BB0, TT2 - VPERM2I128 $0x13, CC0, DD0, TT3 - VPXOR (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3 - VMOVDQU TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup) - MOVQ $128, itr1 - LEAQ 128(inp), inp - SUBQ $128, inl - VPERM2I128 $0x02, AA1, BB1, AA0 - VPERM2I128 $0x02, CC1, DD1, BB0 - VPERM2I128 $0x13, AA1, BB1, CC0 - VPERM2I128 $0x13, CC1, DD1, DD0 - - JMP sealAVX2SealHash - -// ---------------------------------------------------------------------------- -// Special optimization for the last 384 bytes of ciphertext sealAVX2Tail384: - // Need to decrypt up to 384 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA DD0, TT1; VMOVDQA DD1, TT2; VMOVDQA DD2, TT3 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y4, Y7 + VMOVDQA Y1, Y11 + VMOVDQA Y2, Y15 sealAVX2Tail384LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail384LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - polyAdd(0(oup)) - polyMul - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - polyAdd(16(oup)) - polyMul - LEAQ 32(oup), oup - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - DECQ itr1 - JG sealAVX2Tail384LoopA - DECQ itr2 - JGE sealAVX2Tail384LoopB + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + DECQ CX + JG sealAVX2Tail384LoopA + DECQ R9 + JGE sealAVX2Tail384LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD Y7, Y4, Y4 + VPADDD Y11, Y1, Y1 + VPADDD Y15, Y2, Y2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPERM2I128 $0x02, Y12, Y4, Y7 + VPERM2I128 $0x13, Y0, Y14, Y11 + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR (SI), Y3, Y3 + VPXOR 32(SI), Y7, Y7 + VPXOR 64(SI), Y11, Y11 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y3, (DI) + VMOVDQU Y7, 32(DI) + VMOVDQU Y11, 64(DI) + VMOVDQU Y15, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y3 + VPERM2I128 $0x02, Y13, Y1, Y7 + VPERM2I128 $0x13, Y5, Y9, Y11 + VPERM2I128 $0x13, Y13, Y1, Y15 + VPXOR 128(SI), Y3, Y3 + VPXOR 160(SI), Y7, Y7 + VPXOR 192(SI), Y11, Y11 + VPXOR 224(SI), Y15, Y15 + VMOVDQU Y3, 128(DI) + VMOVDQU Y7, 160(DI) + VMOVDQU Y11, 192(DI) + VMOVDQU Y15, 224(DI) + MOVQ $0x00000100, CX + LEAQ 256(SI), SI + SUBQ $0x00000100, BX + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + JMP sealAVX2SealHash - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2 - VPADDD TT1, DD0, DD0; VPADDD TT2, DD1, DD1; VPADDD TT3, DD2, DD2 - VPERM2I128 $0x02, AA0, BB0, TT0 - VPERM2I128 $0x02, CC0, DD0, TT1 - VPERM2I128 $0x13, AA0, BB0, TT2 - VPERM2I128 $0x13, CC0, DD0, TT3 - VPXOR (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3 - VMOVDQU TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, TT0 - VPERM2I128 $0x02, CC1, DD1, TT1 - VPERM2I128 $0x13, AA1, BB1, TT2 - VPERM2I128 $0x13, CC1, DD1, TT3 - VPXOR (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3 - VMOVDQU TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup) - MOVQ $256, itr1 - LEAQ 256(inp), inp - SUBQ $256, inl - VPERM2I128 $0x02, AA2, BB2, AA0 - VPERM2I128 $0x02, CC2, DD2, BB0 - VPERM2I128 $0x13, AA2, BB2, CC0 - VPERM2I128 $0x13, CC2, DD2, DD0 - - JMP sealAVX2SealHash - -// ---------------------------------------------------------------------------- -// Special optimization for the last 512 bytes of ciphertext sealAVX2Tail512: - // Need to decrypt up to 512 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) sealAVX2Tail512LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail512LoopB: - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyAdd(0*8(oup)) - polyMulAVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(2*8(oup)) - polyMulAVX2 - LEAQ (4*8)(oup), oup - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - - DECQ itr1 - JG sealAVX2Tail512LoopA - DECQ itr2 - JGE sealAVX2Tail512LoopB - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 - VPERM2I128 $0x02, AA0, BB0, CC3 - VPXOR (0*32)(inp), CC3, CC3 - VMOVDQU CC3, (0*32)(oup) - VPERM2I128 $0x02, CC0, DD0, CC3 - VPXOR (1*32)(inp), CC3, CC3 - VMOVDQU CC3, (1*32)(oup) - VPERM2I128 $0x13, AA0, BB0, CC3 - VPXOR (2*32)(inp), CC3, CC3 - VMOVDQU CC3, (2*32)(oup) - VPERM2I128 $0x13, CC0, DD0, CC3 - VPXOR (3*32)(inp), CC3, CC3 - VMOVDQU CC3, (3*32)(oup) - - VPERM2I128 $0x02, AA1, BB1, AA0 - VPERM2I128 $0x02, CC1, DD1, BB0 - VPERM2I128 $0x13, AA1, BB1, CC0 - VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) - - VPERM2I128 $0x02, AA2, BB2, AA0 - VPERM2I128 $0x02, CC2, DD2, BB0 - VPERM2I128 $0x13, AA2, BB2, CC0 - VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - - MOVQ $384, itr1 - LEAQ 384(inp), inp - SUBQ $384, inl - VPERM2I128 $0x02, AA3, BB3, AA0 - VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0 - VPERM2I128 $0x13, AA3, BB3, CC0 - VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 - - JMP sealAVX2SealHash + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + DECQ CX + JG sealAVX2Tail512LoopA + DECQ R9 + JGE sealAVX2Tail512LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) + VPERM2I128 $0x02, Y0, Y14, Y15 + VPXOR (SI), Y15, Y15 + VMOVDQU Y15, (DI) + VPERM2I128 $0x02, Y12, Y4, Y15 + VPXOR 32(SI), Y15, Y15 + VMOVDQU Y15, 32(DI) + VPERM2I128 $0x13, Y0, Y14, Y15 + VPXOR 64(SI), Y15, Y15 + VMOVDQU Y15, 64(DI) + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y15, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + MOVQ $0x00000180, CX + LEAQ 384(SI), SI + SUBQ $0x00000180, BX + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + JMP sealAVX2SealHash diff --git a/src/vendor/golang.org/x/crypto/cryptobyte/asn1/asn1.go b/src/vendor/golang.org/x/crypto/cryptobyte/asn1/asn1.go index cda8e3ed..90ef6a24 100644 --- a/src/vendor/golang.org/x/crypto/cryptobyte/asn1/asn1.go +++ b/src/vendor/golang.org/x/crypto/cryptobyte/asn1/asn1.go @@ -4,7 +4,7 @@ // Package asn1 contains supporting types for parsing and building ASN.1 // messages with the cryptobyte package. -package asn1 // import "golang.org/x/crypto/cryptobyte/asn1" +package asn1 // Tag represents an ASN.1 identifier octet, consisting of a tag number // (indicating a type) and class (such as context-specific or constructed). diff --git a/src/vendor/golang.org/x/crypto/cryptobyte/string.go b/src/vendor/golang.org/x/crypto/cryptobyte/string.go index 10692a8a..4b0f8097 100644 --- a/src/vendor/golang.org/x/crypto/cryptobyte/string.go +++ b/src/vendor/golang.org/x/crypto/cryptobyte/string.go @@ -15,7 +15,7 @@ // // See the documentation and examples for the Builder and String types to get // started. -package cryptobyte // import "golang.org/x/crypto/cryptobyte" +package cryptobyte // String represents a string of bytes. It provides methods for parsing // fixed-length and length-prefixed values from it. diff --git a/src/vendor/golang.org/x/crypto/hkdf/hkdf.go b/src/vendor/golang.org/x/crypto/hkdf/hkdf.go deleted file mode 100644 index f4ded5fe..00000000 --- a/src/vendor/golang.org/x/crypto/hkdf/hkdf.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package hkdf implements the HMAC-based Extract-and-Expand Key Derivation -// Function (HKDF) as defined in RFC 5869. -// -// HKDF is a cryptographic key derivation function (KDF) with the goal of -// expanding limited input keying material into one or more cryptographically -// strong secret keys. -package hkdf // import "golang.org/x/crypto/hkdf" - -import ( - "crypto/hmac" - "errors" - "hash" - "io" -) - -// Extract generates a pseudorandom key for use with Expand from an input secret -// and an optional independent salt. -// -// Only use this function if you need to reuse the extracted key with multiple -// Expand invocations and different context values. Most common scenarios, -// including the generation of multiple keys, should use New instead. -func Extract(hash func() hash.Hash, secret, salt []byte) []byte { - if salt == nil { - salt = make([]byte, hash().Size()) - } - extractor := hmac.New(hash, salt) - extractor.Write(secret) - return extractor.Sum(nil) -} - -type hkdf struct { - expander hash.Hash - size int - - info []byte - counter byte - - prev []byte - buf []byte -} - -func (f *hkdf) Read(p []byte) (int, error) { - // Check whether enough data can be generated - need := len(p) - remains := len(f.buf) + int(255-f.counter+1)*f.size - if remains < need { - return 0, errors.New("hkdf: entropy limit reached") - } - // Read any leftover from the buffer - n := copy(p, f.buf) - p = p[n:] - - // Fill the rest of the buffer - for len(p) > 0 { - if f.counter > 1 { - f.expander.Reset() - } - f.expander.Write(f.prev) - f.expander.Write(f.info) - f.expander.Write([]byte{f.counter}) - f.prev = f.expander.Sum(f.prev[:0]) - f.counter++ - - // Copy the new batch into p - f.buf = f.prev - n = copy(p, f.buf) - p = p[n:] - } - // Save leftovers for next run - f.buf = f.buf[n:] - - return need, nil -} - -// Expand returns a Reader, from which keys can be read, using the given -// pseudorandom key and optional context info, skipping the extraction step. -// -// The pseudorandomKey should have been generated by Extract, or be a uniformly -// random or pseudorandom cryptographically strong key. See RFC 5869, Section -// 3.3. Most common scenarios will want to use New instead. -func Expand(hash func() hash.Hash, pseudorandomKey, info []byte) io.Reader { - expander := hmac.New(hash, pseudorandomKey) - return &hkdf{expander, expander.Size(), info, 1, nil, nil} -} - -// New returns a Reader, from which keys can be read, using the given hash, -// secret, salt and context info. Salt and info can be nil. -func New(hash func() hash.Hash, secret, salt, info []byte) io.Reader { - prk := Extract(hash, secret, salt) - return Expand(hash, prk, info) -} diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go b/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go index 333da285..bd896bdc 100644 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!amd64 && !ppc64le && !s390x) || !gc || purego +//go:build (!amd64 && !ppc64le && !ppc64 && !s390x) || !gc || purego package poly1305 diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s index e0d3c647..13375738 100644 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s @@ -1,108 +1,93 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// Code generated by command: go run sum_amd64_asm.go -out ../sum_amd64.s -pkg poly1305. DO NOT EDIT. //go:build gc && !purego -#include "textflag.h" - -#define POLY1305_ADD(msg, h0, h1, h2) \ - ADDQ 0(msg), h0; \ - ADCQ 8(msg), h1; \ - ADCQ $1, h2; \ - LEAQ 16(msg), msg - -#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3) \ - MOVQ r0, AX; \ - MULQ h0; \ - MOVQ AX, t0; \ - MOVQ DX, t1; \ - MOVQ r0, AX; \ - MULQ h1; \ - ADDQ AX, t1; \ - ADCQ $0, DX; \ - MOVQ r0, t2; \ - IMULQ h2, t2; \ - ADDQ DX, t2; \ - \ - MOVQ r1, AX; \ - MULQ h0; \ - ADDQ AX, t1; \ - ADCQ $0, DX; \ - MOVQ DX, h0; \ - MOVQ r1, t3; \ - IMULQ h2, t3; \ - MOVQ r1, AX; \ - MULQ h1; \ - ADDQ AX, t2; \ - ADCQ DX, t3; \ - ADDQ h0, t2; \ - ADCQ $0, t3; \ - \ - MOVQ t0, h0; \ - MOVQ t1, h1; \ - MOVQ t2, h2; \ - ANDQ $3, h2; \ - MOVQ t2, t0; \ - ANDQ $0xFFFFFFFFFFFFFFFC, t0; \ - ADDQ t0, h0; \ - ADCQ t3, h1; \ - ADCQ $0, h2; \ - SHRQ $2, t3, t2; \ - SHRQ $2, t3; \ - ADDQ t2, h0; \ - ADCQ t3, h1; \ - ADCQ $0, h2 - -// func update(state *[7]uint64, msg []byte) +// func update(state *macState, msg []byte) TEXT ·update(SB), $0-32 MOVQ state+0(FP), DI MOVQ msg_base+8(FP), SI MOVQ msg_len+16(FP), R15 - - MOVQ 0(DI), R8 // h0 - MOVQ 8(DI), R9 // h1 - MOVQ 16(DI), R10 // h2 - MOVQ 24(DI), R11 // r0 - MOVQ 32(DI), R12 // r1 - - CMPQ R15, $16 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + CMPQ R15, $0x10 JB bytes_between_0_and_15 loop: - POLY1305_ADD(SI, R8, R9, R10) + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ $0x01, R10 + LEAQ 16(SI), SI multiply: - POLY1305_MUL(R8, R9, R10, R11, R12, BX, CX, R13, R14) - SUBQ $16, R15 - CMPQ R15, $16 - JAE loop + MOVQ R11, AX + MULQ R8 + MOVQ AX, BX + MOVQ DX, CX + MOVQ R11, AX + MULQ R9 + ADDQ AX, CX + ADCQ $0x00, DX + MOVQ R11, R13 + IMULQ R10, R13 + ADDQ DX, R13 + MOVQ R12, AX + MULQ R8 + ADDQ AX, CX + ADCQ $0x00, DX + MOVQ DX, R8 + MOVQ R12, R14 + IMULQ R10, R14 + MOVQ R12, AX + MULQ R9 + ADDQ AX, R13 + ADCQ DX, R14 + ADDQ R8, R13 + ADCQ $0x00, R14 + MOVQ BX, R8 + MOVQ CX, R9 + MOVQ R13, R10 + ANDQ $0x03, R10 + MOVQ R13, BX + ANDQ $-4, BX + ADDQ BX, R8 + ADCQ R14, R9 + ADCQ $0x00, R10 + SHRQ $0x02, R14, R13 + SHRQ $0x02, R14 + ADDQ R13, R8 + ADCQ R14, R9 + ADCQ $0x00, R10 + SUBQ $0x10, R15 + CMPQ R15, $0x10 + JAE loop bytes_between_0_and_15: TESTQ R15, R15 JZ done - MOVQ $1, BX + MOVQ $0x00000001, BX XORQ CX, CX XORQ R13, R13 ADDQ R15, SI flush_buffer: - SHLQ $8, BX, CX - SHLQ $8, BX + SHLQ $0x08, BX, CX + SHLQ $0x08, BX MOVB -1(SI), R13 XORQ R13, BX DECQ SI DECQ R15 JNZ flush_buffer - ADDQ BX, R8 ADCQ CX, R9 - ADCQ $0, R10 - MOVQ $16, R15 + ADCQ $0x00, R10 + MOVQ $0x00000010, R15 JMP multiply done: - MOVQ R8, 0(DI) + MOVQ R8, (DI) MOVQ R9, 8(DI) MOVQ R10, 16(DI) RET diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.go similarity index 95% rename from src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go rename to src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.go index 4aec4874..1a1679aa 100644 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build gc && !purego +//go:build gc && !purego && (ppc64 || ppc64le) package poly1305 diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s similarity index 89% rename from src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s rename to src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s index b3c1699b..6899a1da 100644 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s @@ -2,15 +2,25 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build gc && !purego +//go:build gc && !purego && (ppc64 || ppc64le) #include "textflag.h" // This was ported from the amd64 implementation. +#ifdef GOARCH_ppc64le +#define LE_MOVD MOVD +#define LE_MOVWZ MOVWZ +#define LE_MOVHZ MOVHZ +#else +#define LE_MOVD MOVDBR +#define LE_MOVWZ MOVWBR +#define LE_MOVHZ MOVHBR +#endif + #define POLY1305_ADD(msg, h0, h1, h2, t0, t1, t2) \ - MOVD (msg), t0; \ - MOVD 8(msg), t1; \ + LE_MOVD (msg)( R0), t0; \ + LE_MOVD (msg)(R24), t1; \ MOVD $1, t2; \ ADDC t0, h0, h0; \ ADDE t1, h1, h1; \ @@ -50,10 +60,6 @@ ADDE t3, h1, h1; \ ADDZE h2 -DATA ·poly1305Mask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF -DATA ·poly1305Mask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC -GLOBL ·poly1305Mask<>(SB), RODATA, $16 - // func update(state *[7]uint64, msg []byte) TEXT ·update(SB), $0-32 MOVD state+0(FP), R3 @@ -66,6 +72,8 @@ TEXT ·update(SB), $0-32 MOVD 24(R3), R11 // r0 MOVD 32(R3), R12 // r1 + MOVD $8, R24 + CMP R5, $16 BLT bytes_between_0_and_15 @@ -94,7 +102,7 @@ flush_buffer: // Greater than 8 -- load the rightmost remaining bytes in msg // and put into R17 (h1) - MOVD (R4)(R21), R17 + LE_MOVD (R4)(R21), R17 MOVD $16, R22 // Find the offset to those bytes @@ -118,7 +126,7 @@ just1: BLT less8 // Exactly 8 - MOVD (R4), R16 + LE_MOVD (R4), R16 CMP R17, $0 @@ -133,7 +141,7 @@ less8: MOVD $0, R22 // shift count CMP R5, $4 BLT less4 - MOVWZ (R4), R16 + LE_MOVWZ (R4), R16 ADD $4, R4 ADD $-4, R5 MOVD $32, R22 @@ -141,7 +149,7 @@ less8: less4: CMP R5, $2 BLT less2 - MOVHZ (R4), R21 + LE_MOVHZ (R4), R21 SLD R22, R21, R21 OR R16, R21, R16 ADD $16, R22 diff --git a/src/vendor/golang.org/x/crypto/sha3/doc.go b/src/vendor/golang.org/x/crypto/sha3/doc.go deleted file mode 100644 index decd8cf9..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/doc.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package sha3 implements the SHA-3 fixed-output-length hash functions and -// the SHAKE variable-output-length hash functions defined by FIPS-202. -// -// Both types of hash function use the "sponge" construction and the Keccak -// permutation. For a detailed specification see http://keccak.noekeon.org/ -// -// # Guidance -// -// If you aren't sure what function you need, use SHAKE256 with at least 64 -// bytes of output. The SHAKE instances are faster than the SHA3 instances; -// the latter have to allocate memory to conform to the hash.Hash interface. -// -// If you need a secret-key MAC (message authentication code), prepend the -// secret key to the input, hash with SHAKE256 and read at least 32 bytes of -// output. -// -// # Security strengths -// -// The SHA3-x (x equals 224, 256, 384, or 512) functions have a security -// strength against preimage attacks of x bits. Since they only produce "x" -// bits of output, their collision-resistance is only "x/2" bits. -// -// The SHAKE-256 and -128 functions have a generic security strength of 256 and -// 128 bits against all attacks, provided that at least 2x bits of their output -// is used. Requesting more than 64 or 32 bytes of output, respectively, does -// not increase the collision-resistance of the SHAKE functions. -// -// # The sponge construction -// -// A sponge builds a pseudo-random function from a public pseudo-random -// permutation, by applying the permutation to a state of "rate + capacity" -// bytes, but hiding "capacity" of the bytes. -// -// A sponge starts out with a zero state. To hash an input using a sponge, up -// to "rate" bytes of the input are XORed into the sponge's state. The sponge -// is then "full" and the permutation is applied to "empty" it. This process is -// repeated until all the input has been "absorbed". The input is then padded. -// The digest is "squeezed" from the sponge in the same way, except that output -// is copied out instead of input being XORed in. -// -// A sponge is parameterized by its generic security strength, which is equal -// to half its capacity; capacity + rate is equal to the permutation's width. -// Since the KeccakF-1600 permutation is 1600 bits (200 bytes) wide, this means -// that the security strength of a sponge instance is equal to (1600 - bitrate) / 2. -// -// # Recommendations -// -// The SHAKE functions are recommended for most new uses. They can produce -// output of arbitrary length. SHAKE256, with an output length of at least -// 64 bytes, provides 256-bit security against all attacks. The Keccak team -// recommends it for most applications upgrading from SHA2-512. (NIST chose a -// much stronger, but much slower, sponge instance for SHA3-512.) -// -// The SHA-3 functions are "drop-in" replacements for the SHA-2 functions. -// They produce output of the same length, with the same security strengths -// against all attacks. This means, in particular, that SHA3-256 only has -// 128-bit collision resistance, because its output length is 32 bytes. -package sha3 // import "golang.org/x/crypto/sha3" diff --git a/src/vendor/golang.org/x/crypto/sha3/hashes.go b/src/vendor/golang.org/x/crypto/sha3/hashes.go deleted file mode 100644 index 5eae6cb9..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/hashes.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -// This file provides functions for creating instances of the SHA-3 -// and SHAKE hash functions, as well as utility functions for hashing -// bytes. - -import ( - "hash" -) - -// New224 creates a new SHA3-224 hash. -// Its generic security strength is 224 bits against preimage attacks, -// and 112 bits against collision attacks. -func New224() hash.Hash { - return new224() -} - -// New256 creates a new SHA3-256 hash. -// Its generic security strength is 256 bits against preimage attacks, -// and 128 bits against collision attacks. -func New256() hash.Hash { - return new256() -} - -// New384 creates a new SHA3-384 hash. -// Its generic security strength is 384 bits against preimage attacks, -// and 192 bits against collision attacks. -func New384() hash.Hash { - return new384() -} - -// New512 creates a new SHA3-512 hash. -// Its generic security strength is 512 bits against preimage attacks, -// and 256 bits against collision attacks. -func New512() hash.Hash { - return new512() -} - -func new224Generic() *state { - return &state{rate: 144, outputLen: 28, dsbyte: 0x06} -} - -func new256Generic() *state { - return &state{rate: 136, outputLen: 32, dsbyte: 0x06} -} - -func new384Generic() *state { - return &state{rate: 104, outputLen: 48, dsbyte: 0x06} -} - -func new512Generic() *state { - return &state{rate: 72, outputLen: 64, dsbyte: 0x06} -} - -// NewLegacyKeccak256 creates a new Keccak-256 hash. -// -// Only use this function if you require compatibility with an existing cryptosystem -// that uses non-standard padding. All other users should use New256 instead. -func NewLegacyKeccak256() hash.Hash { return &state{rate: 136, outputLen: 32, dsbyte: 0x01} } - -// NewLegacyKeccak512 creates a new Keccak-512 hash. -// -// Only use this function if you require compatibility with an existing cryptosystem -// that uses non-standard padding. All other users should use New512 instead. -func NewLegacyKeccak512() hash.Hash { return &state{rate: 72, outputLen: 64, dsbyte: 0x01} } - -// Sum224 returns the SHA3-224 digest of the data. -func Sum224(data []byte) (digest [28]byte) { - h := New224() - h.Write(data) - h.Sum(digest[:0]) - return -} - -// Sum256 returns the SHA3-256 digest of the data. -func Sum256(data []byte) (digest [32]byte) { - h := New256() - h.Write(data) - h.Sum(digest[:0]) - return -} - -// Sum384 returns the SHA3-384 digest of the data. -func Sum384(data []byte) (digest [48]byte) { - h := New384() - h.Write(data) - h.Sum(digest[:0]) - return -} - -// Sum512 returns the SHA3-512 digest of the data. -func Sum512(data []byte) (digest [64]byte) { - h := New512() - h.Write(data) - h.Sum(digest[:0]) - return -} diff --git a/src/vendor/golang.org/x/crypto/sha3/hashes_noasm.go b/src/vendor/golang.org/x/crypto/sha3/hashes_noasm.go deleted file mode 100644 index 9d85fb62..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/hashes_noasm.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !gc || purego || !s390x - -package sha3 - -func new224() *state { - return new224Generic() -} - -func new256() *state { - return new256Generic() -} - -func new384() *state { - return new384Generic() -} - -func new512() *state { - return new512Generic() -} diff --git a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.go b/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.go deleted file mode 100644 index b908696b..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build amd64 && !purego && gc - -package sha3 - -// This function is implemented in keccakf_amd64.s. - -//go:noescape - -func keccakF1600(a *[25]uint64) diff --git a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.s b/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.s deleted file mode 100644 index 1f539388..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.s +++ /dev/null @@ -1,390 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build amd64 && !purego && gc - -// This code was translated into a form compatible with 6a from the public -// domain sources at https://github.com/gvanas/KeccakCodePackage - -// Offsets in state -#define _ba (0*8) -#define _be (1*8) -#define _bi (2*8) -#define _bo (3*8) -#define _bu (4*8) -#define _ga (5*8) -#define _ge (6*8) -#define _gi (7*8) -#define _go (8*8) -#define _gu (9*8) -#define _ka (10*8) -#define _ke (11*8) -#define _ki (12*8) -#define _ko (13*8) -#define _ku (14*8) -#define _ma (15*8) -#define _me (16*8) -#define _mi (17*8) -#define _mo (18*8) -#define _mu (19*8) -#define _sa (20*8) -#define _se (21*8) -#define _si (22*8) -#define _so (23*8) -#define _su (24*8) - -// Temporary registers -#define rT1 AX - -// Round vars -#define rpState DI -#define rpStack SP - -#define rDa BX -#define rDe CX -#define rDi DX -#define rDo R8 -#define rDu R9 - -#define rBa R10 -#define rBe R11 -#define rBi R12 -#define rBo R13 -#define rBu R14 - -#define rCa SI -#define rCe BP -#define rCi rBi -#define rCo rBo -#define rCu R15 - -#define MOVQ_RBI_RCE MOVQ rBi, rCe -#define XORQ_RT1_RCA XORQ rT1, rCa -#define XORQ_RT1_RCE XORQ rT1, rCe -#define XORQ_RBA_RCU XORQ rBa, rCu -#define XORQ_RBE_RCU XORQ rBe, rCu -#define XORQ_RDU_RCU XORQ rDu, rCu -#define XORQ_RDA_RCA XORQ rDa, rCa -#define XORQ_RDE_RCE XORQ rDe, rCe - -#define mKeccakRound(iState, oState, rc, B_RBI_RCE, G_RT1_RCA, G_RT1_RCE, G_RBA_RCU, K_RT1_RCA, K_RT1_RCE, K_RBA_RCU, M_RT1_RCA, M_RT1_RCE, M_RBE_RCU, S_RDU_RCU, S_RDA_RCA, S_RDE_RCE) \ - /* Prepare round */ \ - MOVQ rCe, rDa; \ - ROLQ $1, rDa; \ - \ - MOVQ _bi(iState), rCi; \ - XORQ _gi(iState), rDi; \ - XORQ rCu, rDa; \ - XORQ _ki(iState), rCi; \ - XORQ _mi(iState), rDi; \ - XORQ rDi, rCi; \ - \ - MOVQ rCi, rDe; \ - ROLQ $1, rDe; \ - \ - MOVQ _bo(iState), rCo; \ - XORQ _go(iState), rDo; \ - XORQ rCa, rDe; \ - XORQ _ko(iState), rCo; \ - XORQ _mo(iState), rDo; \ - XORQ rDo, rCo; \ - \ - MOVQ rCo, rDi; \ - ROLQ $1, rDi; \ - \ - MOVQ rCu, rDo; \ - XORQ rCe, rDi; \ - ROLQ $1, rDo; \ - \ - MOVQ rCa, rDu; \ - XORQ rCi, rDo; \ - ROLQ $1, rDu; \ - \ - /* Result b */ \ - MOVQ _ba(iState), rBa; \ - MOVQ _ge(iState), rBe; \ - XORQ rCo, rDu; \ - MOVQ _ki(iState), rBi; \ - MOVQ _mo(iState), rBo; \ - MOVQ _su(iState), rBu; \ - XORQ rDe, rBe; \ - ROLQ $44, rBe; \ - XORQ rDi, rBi; \ - XORQ rDa, rBa; \ - ROLQ $43, rBi; \ - \ - MOVQ rBe, rCa; \ - MOVQ rc, rT1; \ - ORQ rBi, rCa; \ - XORQ rBa, rT1; \ - XORQ rT1, rCa; \ - MOVQ rCa, _ba(oState); \ - \ - XORQ rDu, rBu; \ - ROLQ $14, rBu; \ - MOVQ rBa, rCu; \ - ANDQ rBe, rCu; \ - XORQ rBu, rCu; \ - MOVQ rCu, _bu(oState); \ - \ - XORQ rDo, rBo; \ - ROLQ $21, rBo; \ - MOVQ rBo, rT1; \ - ANDQ rBu, rT1; \ - XORQ rBi, rT1; \ - MOVQ rT1, _bi(oState); \ - \ - NOTQ rBi; \ - ORQ rBa, rBu; \ - ORQ rBo, rBi; \ - XORQ rBo, rBu; \ - XORQ rBe, rBi; \ - MOVQ rBu, _bo(oState); \ - MOVQ rBi, _be(oState); \ - B_RBI_RCE; \ - \ - /* Result g */ \ - MOVQ _gu(iState), rBe; \ - XORQ rDu, rBe; \ - MOVQ _ka(iState), rBi; \ - ROLQ $20, rBe; \ - XORQ rDa, rBi; \ - ROLQ $3, rBi; \ - MOVQ _bo(iState), rBa; \ - MOVQ rBe, rT1; \ - ORQ rBi, rT1; \ - XORQ rDo, rBa; \ - MOVQ _me(iState), rBo; \ - MOVQ _si(iState), rBu; \ - ROLQ $28, rBa; \ - XORQ rBa, rT1; \ - MOVQ rT1, _ga(oState); \ - G_RT1_RCA; \ - \ - XORQ rDe, rBo; \ - ROLQ $45, rBo; \ - MOVQ rBi, rT1; \ - ANDQ rBo, rT1; \ - XORQ rBe, rT1; \ - MOVQ rT1, _ge(oState); \ - G_RT1_RCE; \ - \ - XORQ rDi, rBu; \ - ROLQ $61, rBu; \ - MOVQ rBu, rT1; \ - ORQ rBa, rT1; \ - XORQ rBo, rT1; \ - MOVQ rT1, _go(oState); \ - \ - ANDQ rBe, rBa; \ - XORQ rBu, rBa; \ - MOVQ rBa, _gu(oState); \ - NOTQ rBu; \ - G_RBA_RCU; \ - \ - ORQ rBu, rBo; \ - XORQ rBi, rBo; \ - MOVQ rBo, _gi(oState); \ - \ - /* Result k */ \ - MOVQ _be(iState), rBa; \ - MOVQ _gi(iState), rBe; \ - MOVQ _ko(iState), rBi; \ - MOVQ _mu(iState), rBo; \ - MOVQ _sa(iState), rBu; \ - XORQ rDi, rBe; \ - ROLQ $6, rBe; \ - XORQ rDo, rBi; \ - ROLQ $25, rBi; \ - MOVQ rBe, rT1; \ - ORQ rBi, rT1; \ - XORQ rDe, rBa; \ - ROLQ $1, rBa; \ - XORQ rBa, rT1; \ - MOVQ rT1, _ka(oState); \ - K_RT1_RCA; \ - \ - XORQ rDu, rBo; \ - ROLQ $8, rBo; \ - MOVQ rBi, rT1; \ - ANDQ rBo, rT1; \ - XORQ rBe, rT1; \ - MOVQ rT1, _ke(oState); \ - K_RT1_RCE; \ - \ - XORQ rDa, rBu; \ - ROLQ $18, rBu; \ - NOTQ rBo; \ - MOVQ rBo, rT1; \ - ANDQ rBu, rT1; \ - XORQ rBi, rT1; \ - MOVQ rT1, _ki(oState); \ - \ - MOVQ rBu, rT1; \ - ORQ rBa, rT1; \ - XORQ rBo, rT1; \ - MOVQ rT1, _ko(oState); \ - \ - ANDQ rBe, rBa; \ - XORQ rBu, rBa; \ - MOVQ rBa, _ku(oState); \ - K_RBA_RCU; \ - \ - /* Result m */ \ - MOVQ _ga(iState), rBe; \ - XORQ rDa, rBe; \ - MOVQ _ke(iState), rBi; \ - ROLQ $36, rBe; \ - XORQ rDe, rBi; \ - MOVQ _bu(iState), rBa; \ - ROLQ $10, rBi; \ - MOVQ rBe, rT1; \ - MOVQ _mi(iState), rBo; \ - ANDQ rBi, rT1; \ - XORQ rDu, rBa; \ - MOVQ _so(iState), rBu; \ - ROLQ $27, rBa; \ - XORQ rBa, rT1; \ - MOVQ rT1, _ma(oState); \ - M_RT1_RCA; \ - \ - XORQ rDi, rBo; \ - ROLQ $15, rBo; \ - MOVQ rBi, rT1; \ - ORQ rBo, rT1; \ - XORQ rBe, rT1; \ - MOVQ rT1, _me(oState); \ - M_RT1_RCE; \ - \ - XORQ rDo, rBu; \ - ROLQ $56, rBu; \ - NOTQ rBo; \ - MOVQ rBo, rT1; \ - ORQ rBu, rT1; \ - XORQ rBi, rT1; \ - MOVQ rT1, _mi(oState); \ - \ - ORQ rBa, rBe; \ - XORQ rBu, rBe; \ - MOVQ rBe, _mu(oState); \ - \ - ANDQ rBa, rBu; \ - XORQ rBo, rBu; \ - MOVQ rBu, _mo(oState); \ - M_RBE_RCU; \ - \ - /* Result s */ \ - MOVQ _bi(iState), rBa; \ - MOVQ _go(iState), rBe; \ - MOVQ _ku(iState), rBi; \ - XORQ rDi, rBa; \ - MOVQ _ma(iState), rBo; \ - ROLQ $62, rBa; \ - XORQ rDo, rBe; \ - MOVQ _se(iState), rBu; \ - ROLQ $55, rBe; \ - \ - XORQ rDu, rBi; \ - MOVQ rBa, rDu; \ - XORQ rDe, rBu; \ - ROLQ $2, rBu; \ - ANDQ rBe, rDu; \ - XORQ rBu, rDu; \ - MOVQ rDu, _su(oState); \ - \ - ROLQ $39, rBi; \ - S_RDU_RCU; \ - NOTQ rBe; \ - XORQ rDa, rBo; \ - MOVQ rBe, rDa; \ - ANDQ rBi, rDa; \ - XORQ rBa, rDa; \ - MOVQ rDa, _sa(oState); \ - S_RDA_RCA; \ - \ - ROLQ $41, rBo; \ - MOVQ rBi, rDe; \ - ORQ rBo, rDe; \ - XORQ rBe, rDe; \ - MOVQ rDe, _se(oState); \ - S_RDE_RCE; \ - \ - MOVQ rBo, rDi; \ - MOVQ rBu, rDo; \ - ANDQ rBu, rDi; \ - ORQ rBa, rDo; \ - XORQ rBi, rDi; \ - XORQ rBo, rDo; \ - MOVQ rDi, _si(oState); \ - MOVQ rDo, _so(oState) \ - -// func keccakF1600(a *[25]uint64) -TEXT ·keccakF1600(SB), 0, $200-8 - MOVQ a+0(FP), rpState - - // Convert the user state into an internal state - NOTQ _be(rpState) - NOTQ _bi(rpState) - NOTQ _go(rpState) - NOTQ _ki(rpState) - NOTQ _mi(rpState) - NOTQ _sa(rpState) - - // Execute the KeccakF permutation - MOVQ _ba(rpState), rCa - MOVQ _be(rpState), rCe - MOVQ _bu(rpState), rCu - - XORQ _ga(rpState), rCa - XORQ _ge(rpState), rCe - XORQ _gu(rpState), rCu - - XORQ _ka(rpState), rCa - XORQ _ke(rpState), rCe - XORQ _ku(rpState), rCu - - XORQ _ma(rpState), rCa - XORQ _me(rpState), rCe - XORQ _mu(rpState), rCu - - XORQ _sa(rpState), rCa - XORQ _se(rpState), rCe - MOVQ _si(rpState), rDi - MOVQ _so(rpState), rDo - XORQ _su(rpState), rCu - - mKeccakRound(rpState, rpStack, $0x0000000000000001, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x0000000000008082, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x800000000000808a, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x8000000080008000, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x000000000000808b, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x0000000080000001, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x8000000080008081, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x8000000000008009, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x000000000000008a, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x0000000000000088, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x0000000080008009, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x000000008000000a, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x000000008000808b, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x800000000000008b, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x8000000000008089, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x8000000000008003, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x8000000000008002, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x8000000000000080, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x000000000000800a, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x800000008000000a, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x8000000080008081, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x8000000000008080, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpState, rpStack, $0x0000000080000001, MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE) - mKeccakRound(rpStack, rpState, $0x8000000080008008, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP) - - // Revert the internal state to the user state - NOTQ _be(rpState) - NOTQ _bi(rpState) - NOTQ _go(rpState) - NOTQ _ki(rpState) - NOTQ _mi(rpState) - NOTQ _sa(rpState) - - RET diff --git a/src/vendor/golang.org/x/crypto/sha3/register.go b/src/vendor/golang.org/x/crypto/sha3/register.go deleted file mode 100644 index addfd504..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/register.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.4 - -package sha3 - -import ( - "crypto" -) - -func init() { - crypto.RegisterHash(crypto.SHA3_224, New224) - crypto.RegisterHash(crypto.SHA3_256, New256) - crypto.RegisterHash(crypto.SHA3_384, New384) - crypto.RegisterHash(crypto.SHA3_512, New512) -} diff --git a/src/vendor/golang.org/x/crypto/sha3/sha3.go b/src/vendor/golang.org/x/crypto/sha3/sha3.go deleted file mode 100644 index afedde5a..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/sha3.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -// spongeDirection indicates the direction bytes are flowing through the sponge. -type spongeDirection int - -const ( - // spongeAbsorbing indicates that the sponge is absorbing input. - spongeAbsorbing spongeDirection = iota - // spongeSqueezing indicates that the sponge is being squeezed. - spongeSqueezing -) - -const ( - // maxRate is the maximum size of the internal buffer. SHAKE-256 - // currently needs the largest buffer. - maxRate = 168 -) - -type state struct { - // Generic sponge components. - a [25]uint64 // main state of the hash - rate int // the number of bytes of state to use - - // dsbyte contains the "domain separation" bits and the first bit of - // the padding. Sections 6.1 and 6.2 of [1] separate the outputs of the - // SHA-3 and SHAKE functions by appending bitstrings to the message. - // Using a little-endian bit-ordering convention, these are "01" for SHA-3 - // and "1111" for SHAKE, or 00000010b and 00001111b, respectively. Then the - // padding rule from section 5.1 is applied to pad the message to a multiple - // of the rate, which involves adding a "1" bit, zero or more "0" bits, and - // a final "1" bit. We merge the first "1" bit from the padding into dsbyte, - // giving 00000110b (0x06) and 00011111b (0x1f). - // [1] http://csrc.nist.gov/publications/drafts/fips-202/fips_202_draft.pdf - // "Draft FIPS 202: SHA-3 Standard: Permutation-Based Hash and - // Extendable-Output Functions (May 2014)" - dsbyte byte - - i, n int // storage[i:n] is the buffer, i is only used while squeezing - storage [maxRate]byte - - // Specific to SHA-3 and SHAKE. - outputLen int // the default output size in bytes - state spongeDirection // whether the sponge is absorbing or squeezing -} - -// BlockSize returns the rate of sponge underlying this hash function. -func (d *state) BlockSize() int { return d.rate } - -// Size returns the output size of the hash function in bytes. -func (d *state) Size() int { return d.outputLen } - -// Reset clears the internal state by zeroing the sponge state and -// the buffer indexes, and setting Sponge.state to absorbing. -func (d *state) Reset() { - // Zero the permutation's state. - for i := range d.a { - d.a[i] = 0 - } - d.state = spongeAbsorbing - d.i, d.n = 0, 0 -} - -func (d *state) clone() *state { - ret := *d - return &ret -} - -// permute applies the KeccakF-1600 permutation. It handles -// any input-output buffering. -func (d *state) permute() { - switch d.state { - case spongeAbsorbing: - // If we're absorbing, we need to xor the input into the state - // before applying the permutation. - xorIn(d, d.storage[:d.rate]) - d.n = 0 - keccakF1600(&d.a) - case spongeSqueezing: - // If we're squeezing, we need to apply the permutation before - // copying more output. - keccakF1600(&d.a) - d.i = 0 - copyOut(d, d.storage[:d.rate]) - } -} - -// pads appends the domain separation bits in dsbyte, applies -// the multi-bitrate 10..1 padding rule, and permutes the state. -func (d *state) padAndPermute() { - // Pad with this instance's domain-separator bits. We know that there's - // at least one byte of space in d.buf because, if it were full, - // permute would have been called to empty it. dsbyte also contains the - // first one bit for the padding. See the comment in the state struct. - d.storage[d.n] = d.dsbyte - d.n++ - for d.n < d.rate { - d.storage[d.n] = 0 - d.n++ - } - // This adds the final one bit for the padding. Because of the way that - // bits are numbered from the LSB upwards, the final bit is the MSB of - // the last byte. - d.storage[d.rate-1] ^= 0x80 - // Apply the permutation - d.permute() - d.state = spongeSqueezing - d.n = d.rate - copyOut(d, d.storage[:d.rate]) -} - -// Write absorbs more data into the hash's state. It panics if any -// output has already been read. -func (d *state) Write(p []byte) (written int, err error) { - if d.state != spongeAbsorbing { - panic("sha3: Write after Read") - } - written = len(p) - - for len(p) > 0 { - if d.n == 0 && len(p) >= d.rate { - // The fast path; absorb a full "rate" bytes of input and apply the permutation. - xorIn(d, p[:d.rate]) - p = p[d.rate:] - keccakF1600(&d.a) - } else { - // The slow path; buffer the input until we can fill the sponge, and then xor it in. - todo := d.rate - d.n - if todo > len(p) { - todo = len(p) - } - d.n += copy(d.storage[d.n:], p[:todo]) - p = p[todo:] - - // If the sponge is full, apply the permutation. - if d.n == d.rate { - d.permute() - } - } - } - - return -} - -// Read squeezes an arbitrary number of bytes from the sponge. -func (d *state) Read(out []byte) (n int, err error) { - // If we're still absorbing, pad and apply the permutation. - if d.state == spongeAbsorbing { - d.padAndPermute() - } - - n = len(out) - - // Now, do the squeezing. - for len(out) > 0 { - n := copy(out, d.storage[d.i:d.n]) - d.i += n - out = out[n:] - - // Apply the permutation if we've squeezed the sponge dry. - if d.i == d.rate { - d.permute() - } - } - - return -} - -// Sum applies padding to the hash state and then squeezes out the desired -// number of output bytes. It panics if any output has already been read. -func (d *state) Sum(in []byte) []byte { - if d.state != spongeAbsorbing { - panic("sha3: Sum after Read") - } - - // Make a copy of the original hash so that caller can keep writing - // and summing. - dup := d.clone() - hash := make([]byte, dup.outputLen, 64) // explicit cap to allow stack allocation - dup.Read(hash) - return append(in, hash...) -} diff --git a/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.go b/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.go deleted file mode 100644 index 00d8034a..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build gc && !purego - -package sha3 - -// This file contains code for using the 'compute intermediate -// message digest' (KIMD) and 'compute last message digest' (KLMD) -// instructions to compute SHA-3 and SHAKE hashes on IBM Z. - -import ( - "hash" - - "golang.org/x/sys/cpu" -) - -// codes represent 7-bit KIMD/KLMD function codes as defined in -// the Principles of Operation. -type code uint64 - -const ( - // function codes for KIMD/KLMD - sha3_224 code = 32 - sha3_256 = 33 - sha3_384 = 34 - sha3_512 = 35 - shake_128 = 36 - shake_256 = 37 - nopad = 0x100 -) - -// kimd is a wrapper for the 'compute intermediate message digest' instruction. -// src must be a multiple of the rate for the given function code. -// -//go:noescape -func kimd(function code, chain *[200]byte, src []byte) - -// klmd is a wrapper for the 'compute last message digest' instruction. -// src padding is handled by the instruction. -// -//go:noescape -func klmd(function code, chain *[200]byte, dst, src []byte) - -type asmState struct { - a [200]byte // 1600 bit state - buf []byte // care must be taken to ensure cap(buf) is a multiple of rate - rate int // equivalent to block size - storage [3072]byte // underlying storage for buf - outputLen int // output length for full security - function code // KIMD/KLMD function code - state spongeDirection // whether the sponge is absorbing or squeezing -} - -func newAsmState(function code) *asmState { - var s asmState - s.function = function - switch function { - case sha3_224: - s.rate = 144 - s.outputLen = 28 - case sha3_256: - s.rate = 136 - s.outputLen = 32 - case sha3_384: - s.rate = 104 - s.outputLen = 48 - case sha3_512: - s.rate = 72 - s.outputLen = 64 - case shake_128: - s.rate = 168 - s.outputLen = 32 - case shake_256: - s.rate = 136 - s.outputLen = 64 - default: - panic("sha3: unrecognized function code") - } - - // limit s.buf size to a multiple of s.rate - s.resetBuf() - return &s -} - -func (s *asmState) clone() *asmState { - c := *s - c.buf = c.storage[:len(s.buf):cap(s.buf)] - return &c -} - -// copyIntoBuf copies b into buf. It will panic if there is not enough space to -// store all of b. -func (s *asmState) copyIntoBuf(b []byte) { - bufLen := len(s.buf) - s.buf = s.buf[:len(s.buf)+len(b)] - copy(s.buf[bufLen:], b) -} - -// resetBuf points buf at storage, sets the length to 0 and sets cap to be a -// multiple of the rate. -func (s *asmState) resetBuf() { - max := (cap(s.storage) / s.rate) * s.rate - s.buf = s.storage[:0:max] -} - -// Write (via the embedded io.Writer interface) adds more data to the running hash. -// It never returns an error. -func (s *asmState) Write(b []byte) (int, error) { - if s.state != spongeAbsorbing { - panic("sha3: Write after Read") - } - length := len(b) - for len(b) > 0 { - if len(s.buf) == 0 && len(b) >= cap(s.buf) { - // Hash the data directly and push any remaining bytes - // into the buffer. - remainder := len(b) % s.rate - kimd(s.function, &s.a, b[:len(b)-remainder]) - if remainder != 0 { - s.copyIntoBuf(b[len(b)-remainder:]) - } - return length, nil - } - - if len(s.buf) == cap(s.buf) { - // flush the buffer - kimd(s.function, &s.a, s.buf) - s.buf = s.buf[:0] - } - - // copy as much as we can into the buffer - n := len(b) - if len(b) > cap(s.buf)-len(s.buf) { - n = cap(s.buf) - len(s.buf) - } - s.copyIntoBuf(b[:n]) - b = b[n:] - } - return length, nil -} - -// Read squeezes an arbitrary number of bytes from the sponge. -func (s *asmState) Read(out []byte) (n int, err error) { - // The 'compute last message digest' instruction only stores the digest - // at the first operand (dst) for SHAKE functions. - if s.function != shake_128 && s.function != shake_256 { - panic("sha3: can only call Read for SHAKE functions") - } - - n = len(out) - - // need to pad if we were absorbing - if s.state == spongeAbsorbing { - s.state = spongeSqueezing - - // write hash directly into out if possible - if len(out)%s.rate == 0 { - klmd(s.function, &s.a, out, s.buf) // len(out) may be 0 - s.buf = s.buf[:0] - return - } - - // write hash into buffer - max := cap(s.buf) - if max > len(out) { - max = (len(out)/s.rate)*s.rate + s.rate - } - klmd(s.function, &s.a, s.buf[:max], s.buf) - s.buf = s.buf[:max] - } - - for len(out) > 0 { - // flush the buffer - if len(s.buf) != 0 { - c := copy(out, s.buf) - out = out[c:] - s.buf = s.buf[c:] - continue - } - - // write hash directly into out if possible - if len(out)%s.rate == 0 { - klmd(s.function|nopad, &s.a, out, nil) - return - } - - // write hash into buffer - s.resetBuf() - if cap(s.buf) > len(out) { - s.buf = s.buf[:(len(out)/s.rate)*s.rate+s.rate] - } - klmd(s.function|nopad, &s.a, s.buf, nil) - } - return -} - -// Sum appends the current hash to b and returns the resulting slice. -// It does not change the underlying hash state. -func (s *asmState) Sum(b []byte) []byte { - if s.state != spongeAbsorbing { - panic("sha3: Sum after Read") - } - - // Copy the state to preserve the original. - a := s.a - - // Hash the buffer. Note that we don't clear it because we - // aren't updating the state. - switch s.function { - case sha3_224, sha3_256, sha3_384, sha3_512: - klmd(s.function, &a, nil, s.buf) - return append(b, a[:s.outputLen]...) - case shake_128, shake_256: - d := make([]byte, s.outputLen, 64) - klmd(s.function, &a, d, s.buf) - return append(b, d[:s.outputLen]...) - default: - panic("sha3: unknown function") - } -} - -// Reset resets the Hash to its initial state. -func (s *asmState) Reset() { - for i := range s.a { - s.a[i] = 0 - } - s.resetBuf() - s.state = spongeAbsorbing -} - -// Size returns the number of bytes Sum will return. -func (s *asmState) Size() int { - return s.outputLen -} - -// BlockSize returns the hash's underlying block size. -// The Write method must be able to accept any amount -// of data, but it may operate more efficiently if all writes -// are a multiple of the block size. -func (s *asmState) BlockSize() int { - return s.rate -} - -// Clone returns a copy of the ShakeHash in its current state. -func (s *asmState) Clone() ShakeHash { - return s.clone() -} - -// new224 returns an assembly implementation of SHA3-224 if available, -// otherwise it returns a generic implementation. -func new224() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_224) - } - return new224Generic() -} - -// new256 returns an assembly implementation of SHA3-256 if available, -// otherwise it returns a generic implementation. -func new256() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_256) - } - return new256Generic() -} - -// new384 returns an assembly implementation of SHA3-384 if available, -// otherwise it returns a generic implementation. -func new384() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_384) - } - return new384Generic() -} - -// new512 returns an assembly implementation of SHA3-512 if available, -// otherwise it returns a generic implementation. -func new512() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_512) - } - return new512Generic() -} - -// newShake128 returns an assembly implementation of SHAKE-128 if available, -// otherwise it returns a generic implementation. -func newShake128() ShakeHash { - if cpu.S390X.HasSHA3 { - return newAsmState(shake_128) - } - return newShake128Generic() -} - -// newShake256 returns an assembly implementation of SHAKE-256 if available, -// otherwise it returns a generic implementation. -func newShake256() ShakeHash { - if cpu.S390X.HasSHA3 { - return newAsmState(shake_256) - } - return newShake256Generic() -} diff --git a/src/vendor/golang.org/x/crypto/sha3/shake.go b/src/vendor/golang.org/x/crypto/sha3/shake.go deleted file mode 100644 index 1ea9275b..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/shake.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -// This file defines the ShakeHash interface, and provides -// functions for creating SHAKE and cSHAKE instances, as well as utility -// functions for hashing bytes to arbitrary-length output. -// -// -// SHAKE implementation is based on FIPS PUB 202 [1] -// cSHAKE implementations is based on NIST SP 800-185 [2] -// -// [1] https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf -// [2] https://doi.org/10.6028/NIST.SP.800-185 - -import ( - "encoding/binary" - "hash" - "io" -) - -// ShakeHash defines the interface to hash functions that support -// arbitrary-length output. When used as a plain [hash.Hash], it -// produces minimum-length outputs that provide full-strength generic -// security. -type ShakeHash interface { - hash.Hash - - // Read reads more output from the hash; reading affects the hash's - // state. (ShakeHash.Read is thus very different from Hash.Sum) - // It never returns an error, but subsequent calls to Write or Sum - // will panic. - io.Reader - - // Clone returns a copy of the ShakeHash in its current state. - Clone() ShakeHash -} - -// cSHAKE specific context -type cshakeState struct { - *state // SHA-3 state context and Read/Write operations - - // initBlock is the cSHAKE specific initialization set of bytes. It is initialized - // by newCShake function and stores concatenation of N followed by S, encoded - // by the method specified in 3.3 of [1]. - // It is stored here in order for Reset() to be able to put context into - // initial state. - initBlock []byte -} - -// Consts for configuring initial SHA-3 state -const ( - dsbyteShake = 0x1f - dsbyteCShake = 0x04 - rate128 = 168 - rate256 = 136 -) - -func bytepad(input []byte, w int) []byte { - // leftEncode always returns max 9 bytes - buf := make([]byte, 0, 9+len(input)+w) - buf = append(buf, leftEncode(uint64(w))...) - buf = append(buf, input...) - padlen := w - (len(buf) % w) - return append(buf, make([]byte, padlen)...) -} - -func leftEncode(value uint64) []byte { - var b [9]byte - binary.BigEndian.PutUint64(b[1:], value) - // Trim all but last leading zero bytes - i := byte(1) - for i < 8 && b[i] == 0 { - i++ - } - // Prepend number of encoded bytes - b[i-1] = 9 - i - return b[i-1:] -} - -func newCShake(N, S []byte, rate, outputLen int, dsbyte byte) ShakeHash { - c := cshakeState{state: &state{rate: rate, outputLen: outputLen, dsbyte: dsbyte}} - - // leftEncode returns max 9 bytes - c.initBlock = make([]byte, 0, 9*2+len(N)+len(S)) - c.initBlock = append(c.initBlock, leftEncode(uint64(len(N)*8))...) - c.initBlock = append(c.initBlock, N...) - c.initBlock = append(c.initBlock, leftEncode(uint64(len(S)*8))...) - c.initBlock = append(c.initBlock, S...) - c.Write(bytepad(c.initBlock, c.rate)) - return &c -} - -// Reset resets the hash to initial state. -func (c *cshakeState) Reset() { - c.state.Reset() - c.Write(bytepad(c.initBlock, c.rate)) -} - -// Clone returns copy of a cSHAKE context within its current state. -func (c *cshakeState) Clone() ShakeHash { - b := make([]byte, len(c.initBlock)) - copy(b, c.initBlock) - return &cshakeState{state: c.clone(), initBlock: b} -} - -// Clone returns copy of SHAKE context within its current state. -func (c *state) Clone() ShakeHash { - return c.clone() -} - -// NewShake128 creates a new SHAKE128 variable-output-length ShakeHash. -// Its generic security strength is 128 bits against all attacks if at -// least 32 bytes of its output are used. -func NewShake128() ShakeHash { - return newShake128() -} - -// NewShake256 creates a new SHAKE256 variable-output-length ShakeHash. -// Its generic security strength is 256 bits against all attacks if -// at least 64 bytes of its output are used. -func NewShake256() ShakeHash { - return newShake256() -} - -func newShake128Generic() *state { - return &state{rate: rate128, outputLen: 32, dsbyte: dsbyteShake} -} - -func newShake256Generic() *state { - return &state{rate: rate256, outputLen: 64, dsbyte: dsbyteShake} -} - -// NewCShake128 creates a new instance of cSHAKE128 variable-output-length ShakeHash, -// a customizable variant of SHAKE128. -// N is used to define functions based on cSHAKE, it can be empty when plain cSHAKE is -// desired. S is a customization byte string used for domain separation - two cSHAKE -// computations on same input with different S yield unrelated outputs. -// When N and S are both empty, this is equivalent to NewShake128. -func NewCShake128(N, S []byte) ShakeHash { - if len(N) == 0 && len(S) == 0 { - return NewShake128() - } - return newCShake(N, S, rate128, 32, dsbyteCShake) -} - -// NewCShake256 creates a new instance of cSHAKE256 variable-output-length ShakeHash, -// a customizable variant of SHAKE256. -// N is used to define functions based on cSHAKE, it can be empty when plain cSHAKE is -// desired. S is a customization byte string used for domain separation - two cSHAKE -// computations on same input with different S yield unrelated outputs. -// When N and S are both empty, this is equivalent to NewShake256. -func NewCShake256(N, S []byte) ShakeHash { - if len(N) == 0 && len(S) == 0 { - return NewShake256() - } - return newCShake(N, S, rate256, 64, dsbyteCShake) -} - -// ShakeSum128 writes an arbitrary-length digest of data into hash. -func ShakeSum128(hash, data []byte) { - h := NewShake128() - h.Write(data) - h.Read(hash) -} - -// ShakeSum256 writes an arbitrary-length digest of data into hash. -func ShakeSum256(hash, data []byte) { - h := NewShake256() - h.Write(data) - h.Read(hash) -} diff --git a/src/vendor/golang.org/x/crypto/sha3/shake_noasm.go b/src/vendor/golang.org/x/crypto/sha3/shake_noasm.go deleted file mode 100644 index 4276ba4a..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/shake_noasm.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !gc || purego || !s390x - -package sha3 - -func newShake128() *state { - return newShake128Generic() -} - -func newShake256() *state { - return newShake256Generic() -} diff --git a/src/vendor/golang.org/x/crypto/sha3/xor.go b/src/vendor/golang.org/x/crypto/sha3/xor.go deleted file mode 100644 index 6ada5c95..00000000 --- a/src/vendor/golang.org/x/crypto/sha3/xor.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -import ( - "crypto/subtle" - "encoding/binary" - "unsafe" - - "golang.org/x/sys/cpu" -) - -// xorIn xors the bytes in buf into the state. -func xorIn(d *state, buf []byte) { - if cpu.IsBigEndian { - for i := 0; len(buf) >= 8; i++ { - a := binary.LittleEndian.Uint64(buf) - d.a[i] ^= a - buf = buf[8:] - } - } else { - ab := (*[25 * 64 / 8]byte)(unsafe.Pointer(&d.a)) - subtle.XORBytes(ab[:], ab[:], buf) - } -} - -// copyOut copies uint64s to a byte buffer. -func copyOut(d *state, b []byte) { - if cpu.IsBigEndian { - for i := 0; len(b) >= 8; i++ { - binary.LittleEndian.PutUint64(b, d.a[i]) - b = b[8:] - } - } else { - ab := (*[25 * 64 / 8]byte)(unsafe.Pointer(&d.a)) - copy(b, ab[:]) - } -} diff --git a/src/vendor/golang.org/x/net/LICENSE b/src/vendor/golang.org/x/net/LICENSE index 6a66aea5..2a7cf70d 100644 --- a/src/vendor/golang.org/x/net/LICENSE +++ b/src/vendor/golang.org/x/net/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/vendor/golang.org/x/net/route/address.go b/src/vendor/golang.org/x/net/route/address.go index 5443d672..b649f431 100644 --- a/src/vendor/golang.org/x/net/route/address.go +++ b/src/vendor/golang.org/x/net/route/address.go @@ -170,20 +170,37 @@ func (a *Inet6Addr) marshal(b []byte) (int, error) { // parseInetAddr parses b as an internet address for IPv4 or IPv6. func parseInetAddr(af int, b []byte) (Addr, error) { + const ( + off4 = 4 // offset of in_addr + off6 = 8 // offset of in6_addr + ) switch af { case syscall.AF_INET: - if len(b) < sizeofSockaddrInet { + if len(b) < (off4+1) || len(b) < int(b[0]) || b[0] == 0 { return nil, errInvalidAddr } + sockAddrLen := int(b[0]) a := &Inet4Addr{} - copy(a.IP[:], b[4:8]) + n := off4 + 4 + if sockAddrLen < n { + n = sockAddrLen + } + copy(a.IP[:], b[off4:n]) return a, nil case syscall.AF_INET6: - if len(b) < sizeofSockaddrInet6 { + if len(b) < (off6+1) || len(b) < int(b[0]) || b[0] == 0 { return nil, errInvalidAddr } - a := &Inet6Addr{ZoneID: int(nativeEndian.Uint32(b[24:28]))} - copy(a.IP[:], b[8:24]) + sockAddrLen := int(b[0]) + n := off6 + 16 + if sockAddrLen < n { + n = sockAddrLen + } + a := &Inet6Addr{} + if sockAddrLen == sizeofSockaddrInet6 { + a.ZoneID = int(nativeEndian.Uint32(b[24:28])) + } + copy(a.IP[:], b[off6:n]) if a.IP[0] == 0xfe && a.IP[1]&0xc0 == 0x80 || a.IP[0] == 0xff && (a.IP[1]&0x0f == 0x01 || a.IP[1]&0x0f == 0x02) { // KAME based IPv6 protocol stack usually // embeds the interface index in the @@ -387,12 +404,16 @@ func parseAddrs(attrs uint, fn func(int, []byte) (int, Addr, error), b []byte) ( } b = b[l:] case syscall.AF_INET, syscall.AF_INET6: - af = int(b[1]) - a, err := parseInetAddr(af, b) - if err != nil { - return nil, err + // #70528: if the sockaddrlen is 0, no address to parse inside, + // skip over the record. + if b[0] > 0 { + af = int(b[1]) + a, err := parseInetAddr(af, b) + if err != nil { + return nil, err + } + as[i] = a } - as[i] = a l := roundup(int(b[0])) if len(b) < l { return nil, errMessageTooShort diff --git a/src/vendor/golang.org/x/sys/LICENSE b/src/vendor/golang.org/x/sys/LICENSE index 6a66aea5..2a7cf70d 100644 --- a/src/vendor/golang.org/x/sys/LICENSE +++ b/src/vendor/golang.org/x/sys/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s b/src/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s new file mode 100644 index 00000000..ec2acfe5 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && amd64 && gc + +#include "textflag.h" + +TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctl(SB) +GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) + +TEXT libc_sysctlbyname_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctlbyname(SB) +GLOBL ·libc_sysctlbyname_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctlbyname_trampoline_addr(SB)/8, $libc_sysctlbyname_trampoline<>(SB) diff --git a/src/vendor/golang.org/x/sys/cpu/cpu.go b/src/vendor/golang.org/x/sys/cpu/cpu.go index 8fa707aa..02609d5b 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu.go @@ -105,6 +105,8 @@ var ARM64 struct { HasSVE bool // Scalable Vector Extensions HasSVE2 bool // Scalable Vector Extensions 2 HasASIMDFHM bool // Advanced SIMD multiplication FP16 to FP32 + HasDIT bool // Data Independent Timing support + HasI8MM bool // Advanced SIMD Int8 matrix multiplication instructions _ CacheLinePad } @@ -199,6 +201,25 @@ var S390X struct { _ CacheLinePad } +// RISCV64 contains the supported CPU features and performance characteristics for riscv64 +// platforms. The booleans in RISCV64, with the exception of HasFastMisaligned, indicate +// the presence of RISC-V extensions. +// +// It is safe to assume that all the RV64G extensions are supported and so they are omitted from +// this structure. As riscv64 Go programs require at least RV64G, the code that populates +// this structure cannot run successfully if some of the RV64G extensions are missing. +// The struct is padded to avoid false sharing. +var RISCV64 struct { + _ CacheLinePad + HasFastMisaligned bool // Fast misaligned accesses + HasC bool // Compressed instruction-set extension + HasV bool // Vector extension compatible with RVV 1.0 + HasZba bool // Address generation instructions extension + HasZbb bool // Basic bit-manipulation extension + HasZbs bool // Single-bit instructions extension + _ CacheLinePad +} + func init() { archInit() initOptions() diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_arm64.go b/src/vendor/golang.org/x/sys/cpu/cpu_arm64.go index 0e27a21e..af2aa99f 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_arm64.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_arm64.go @@ -38,6 +38,8 @@ func initOptions() { {Name: "dcpop", Feature: &ARM64.HasDCPOP}, {Name: "asimddp", Feature: &ARM64.HasASIMDDP}, {Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM}, + {Name: "dit", Feature: &ARM64.HasDIT}, + {Name: "i8mm", Feature: &ARM64.HasI8MM}, } } @@ -145,6 +147,11 @@ func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) { ARM64.HasLRCPC = true } + switch extractBits(isar1, 52, 55) { + case 1: + ARM64.HasI8MM = true + } + // ID_AA64PFR0_EL1 switch extractBits(pfr0, 16, 19) { case 0: @@ -168,6 +175,11 @@ func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) { parseARM64SVERegister(getzfr0()) } + + switch extractBits(pfr0, 48, 51) { + case 1: + ARM64.HasDIT = true + } } func parseARM64SVERegister(zfr0 uint64) { diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go new file mode 100644 index 00000000..b838cb9e --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && amd64 && gc + +package cpu + +// darwinSupportsAVX512 checks Darwin kernel for AVX512 support via sysctl +// call (see issue 43089). It also restricts AVX512 support for Darwin to +// kernel version 21.3.0 (MacOS 12.2.0) or later (see issue 49233). +// +// Background: +// Darwin implements a special mechanism to economize on thread state when +// AVX512 specific registers are not in use. This scheme minimizes state when +// preempting threads that haven't yet used any AVX512 instructions, but adds +// special requirements to check for AVX512 hardware support at runtime (e.g. +// via sysctl call or commpage inspection). See issue 43089 and link below for +// full background: +// https://github.com/apple-oss-distributions/xnu/blob/xnu-11215.1.10/osfmk/i386/fpu.c#L214-L240 +// +// Additionally, all versions of the Darwin kernel from 19.6.0 through 21.2.0 +// (corresponding to MacOS 10.15.6 - 12.1) have a bug that can cause corruption +// of the AVX512 mask registers (K0-K7) upon signal return. For this reason +// AVX512 is considered unsafe to use on Darwin for kernel versions prior to +// 21.3.0, where a fix has been confirmed. See issue 49233 for full background. +func darwinSupportsAVX512() bool { + return darwinSysctlEnabled([]byte("hw.optional.avx512f\x00")) && darwinKernelVersionCheck(21, 3, 0) +} + +// Ensure Darwin kernel version is at least major.minor.patch, avoiding dependencies +func darwinKernelVersionCheck(major, minor, patch int) bool { + var release [256]byte + err := darwinOSRelease(&release) + if err != nil { + return false + } + + var mmp [3]int + c := 0 +Loop: + for _, b := range release[:] { + switch { + case b >= '0' && b <= '9': + mmp[c] = 10*mmp[c] + int(b-'0') + case b == '.': + c++ + if c > 2 { + return false + } + case b == 0: + break Loop + default: + return false + } + } + if c != 2 { + return false + } + return mmp[0] > major || mmp[0] == major && (mmp[1] > minor || mmp[1] == minor && mmp[2] >= patch) +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go index 910728fb..32a44514 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go @@ -6,10 +6,10 @@ package cpu -// cpuid is implemented in cpu_x86.s for gc compiler +// cpuid is implemented in cpu_gc_x86.s for gc compiler // and in cpu_gccgo.c for gccgo. func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) -// xgetbv with ecx = 0 is implemented in cpu_x86.s for gc compiler +// xgetbv with ecx = 0 is implemented in cpu_gc_x86.s for gc compiler // and in cpu_gccgo.c for gccgo. func xgetbv() (eax, edx uint32) diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_x86.s b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s similarity index 94% rename from src/vendor/golang.org/x/sys/cpu/cpu_x86.s rename to src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s index 7d7ba33e..ce208ce6 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_x86.s +++ b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s @@ -18,7 +18,7 @@ TEXT ·cpuid(SB), NOSPLIT, $0-24 RET // func xgetbv() (eax, edx uint32) -TEXT ·xgetbv(SB),NOSPLIT,$0-8 +TEXT ·xgetbv(SB), NOSPLIT, $0-8 MOVL $0, CX XGETBV MOVL AX, eax+0(FP) diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go index 99c60fe9..170d21dd 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go @@ -23,9 +23,3 @@ func xgetbv() (eax, edx uint32) { gccgoXgetbv(&a, &d) return a, d } - -// gccgo doesn't build on Darwin, per: -// https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/gcc.rb#L76 -func darwinSupportsAVX512() bool { - return false -} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go b/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go index 3d386d0f..f1caf0f7 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go @@ -35,8 +35,10 @@ const ( hwcap_SHA512 = 1 << 21 hwcap_SVE = 1 << 22 hwcap_ASIMDFHM = 1 << 23 + hwcap_DIT = 1 << 24 hwcap2_SVE2 = 1 << 1 + hwcap2_I8MM = 1 << 13 ) // linuxKernelCanEmulateCPUID reports whether we're running @@ -106,9 +108,11 @@ func doinit() { ARM64.HasSHA512 = isSet(hwCap, hwcap_SHA512) ARM64.HasSVE = isSet(hwCap, hwcap_SVE) ARM64.HasASIMDFHM = isSet(hwCap, hwcap_ASIMDFHM) + ARM64.HasDIT = isSet(hwCap, hwcap_DIT) // HWCAP2 feature bits ARM64.HasSVE2 = isSet(hwCap2, hwcap2_SVE2) + ARM64.HasI8MM = isSet(hwCap2, hwcap2_I8MM) } func isSet(hwc uint, value uint) bool { diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go b/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go index cd63e733..7d902b68 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x +//go:build linux && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x && !riscv64 package cpu diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go b/src/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go new file mode 100644 index 00000000..cb4a0c57 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go @@ -0,0 +1,137 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "syscall" + "unsafe" +) + +// RISC-V extension discovery code for Linux. The approach here is to first try the riscv_hwprobe +// syscall falling back to HWCAP to check for the C extension if riscv_hwprobe is not available. +// +// A note on detection of the Vector extension using HWCAP. +// +// Support for the Vector extension version 1.0 was added to the Linux kernel in release 6.5. +// Support for the riscv_hwprobe syscall was added in 6.4. It follows that if the riscv_hwprobe +// syscall is not available then neither is the Vector extension (which needs kernel support). +// The riscv_hwprobe syscall should then be all we need to detect the Vector extension. +// However, some RISC-V board manufacturers ship boards with an older kernel on top of which +// they have back-ported various versions of the Vector extension patches but not the riscv_hwprobe +// patches. These kernels advertise support for the Vector extension using HWCAP. Falling +// back to HWCAP to detect the Vector extension, if riscv_hwprobe is not available, or simply not +// bothering with riscv_hwprobe at all and just using HWCAP may then seem like an attractive option. +// +// Unfortunately, simply checking the 'V' bit in AT_HWCAP will not work as this bit is used by +// RISC-V board and cloud instance providers to mean different things. The Lichee Pi 4A board +// and the Scaleway RV1 cloud instances use the 'V' bit to advertise their support for the unratified +// 0.7.1 version of the Vector Specification. The Banana Pi BPI-F3 and the CanMV-K230 board use +// it to advertise support for 1.0 of the Vector extension. Versions 0.7.1 and 1.0 of the Vector +// extension are binary incompatible. HWCAP can then not be used in isolation to populate the +// HasV field as this field indicates that the underlying CPU is compatible with RVV 1.0. +// +// There is a way at runtime to distinguish between versions 0.7.1 and 1.0 of the Vector +// specification by issuing a RVV 1.0 vsetvli instruction and checking the vill bit of the vtype +// register. This check would allow us to safely detect version 1.0 of the Vector extension +// with HWCAP, if riscv_hwprobe were not available. However, the check cannot +// be added until the assembler supports the Vector instructions. +// +// Note the riscv_hwprobe syscall does not suffer from these ambiguities by design as all of the +// extensions it advertises support for are explicitly versioned. It's also worth noting that +// the riscv_hwprobe syscall is the only way to detect multi-letter RISC-V extensions, e.g., Zba. +// These cannot be detected using HWCAP and so riscv_hwprobe must be used to detect the majority +// of RISC-V extensions. +// +// Please see https://docs.kernel.org/arch/riscv/hwprobe.html for more information. + +// golang.org/x/sys/cpu is not allowed to depend on golang.org/x/sys/unix so we must +// reproduce the constants, types and functions needed to make the riscv_hwprobe syscall +// here. + +const ( + // Copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. + riscv_HWPROBE_KEY_IMA_EXT_0 = 0x4 + riscv_HWPROBE_IMA_C = 0x2 + riscv_HWPROBE_IMA_V = 0x4 + riscv_HWPROBE_EXT_ZBA = 0x8 + riscv_HWPROBE_EXT_ZBB = 0x10 + riscv_HWPROBE_EXT_ZBS = 0x20 + riscv_HWPROBE_KEY_CPUPERF_0 = 0x5 + riscv_HWPROBE_MISALIGNED_FAST = 0x3 + riscv_HWPROBE_MISALIGNED_MASK = 0x7 +) + +const ( + // sys_RISCV_HWPROBE is copied from golang.org/x/sys/unix/zsysnum_linux_riscv64.go. + sys_RISCV_HWPROBE = 258 +) + +// riscvHWProbePairs is copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. +type riscvHWProbePairs struct { + key int64 + value uint64 +} + +const ( + // CPU features + hwcap_RISCV_ISA_C = 1 << ('C' - 'A') +) + +func doinit() { + // A slice of key/value pair structures is passed to the RISCVHWProbe syscall. The key + // field should be initialised with one of the key constants defined above, e.g., + // RISCV_HWPROBE_KEY_IMA_EXT_0. The syscall will set the value field to the appropriate value. + // If the kernel does not recognise a key it will set the key field to -1 and the value field to 0. + + pairs := []riscvHWProbePairs{ + {riscv_HWPROBE_KEY_IMA_EXT_0, 0}, + {riscv_HWPROBE_KEY_CPUPERF_0, 0}, + } + + // This call only indicates that extensions are supported if they are implemented on all cores. + if riscvHWProbe(pairs, 0) { + if pairs[0].key != -1 { + v := uint(pairs[0].value) + RISCV64.HasC = isSet(v, riscv_HWPROBE_IMA_C) + RISCV64.HasV = isSet(v, riscv_HWPROBE_IMA_V) + RISCV64.HasZba = isSet(v, riscv_HWPROBE_EXT_ZBA) + RISCV64.HasZbb = isSet(v, riscv_HWPROBE_EXT_ZBB) + RISCV64.HasZbs = isSet(v, riscv_HWPROBE_EXT_ZBS) + } + if pairs[1].key != -1 { + v := pairs[1].value & riscv_HWPROBE_MISALIGNED_MASK + RISCV64.HasFastMisaligned = v == riscv_HWPROBE_MISALIGNED_FAST + } + } + + // Let's double check with HWCAP if the C extension does not appear to be supported. + // This may happen if we're running on a kernel older than 6.4. + + if !RISCV64.HasC { + RISCV64.HasC = isSet(hwCap, hwcap_RISCV_ISA_C) + } +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} + +// riscvHWProbe is a simplified version of the generated wrapper function found in +// golang.org/x/sys/unix/zsyscall_linux_riscv64.go. We simplify it by removing the +// cpuCount and cpus parameters which we do not need. We always want to pass 0 for +// these parameters here so the kernel only reports the extensions that are present +// on all cores. +func riscvHWProbe(pairs []riscvHWProbePairs, flags uint) bool { + var _zero uintptr + var p0 unsafe.Pointer + if len(pairs) > 0 { + p0 = unsafe.Pointer(&pairs[0]) + } else { + p0 = unsafe.Pointer(&_zero) + } + + _, _, e1 := syscall.Syscall6(sys_RISCV_HWPROBE, uintptr(p0), uintptr(len(pairs)), uintptr(0), uintptr(0), uintptr(flags), 0) + return e1 == 0 +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_other_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_other_x86.go new file mode 100644 index 00000000..a0fd7e2f --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_other_x86.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64p32 || (amd64 && (!darwin || !gc)) + +package cpu + +func darwinSupportsAVX512() bool { + panic("only implemented for gc && amd64 && darwin") +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go b/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go index 7f0c79c0..aca3199c 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go @@ -8,4 +8,13 @@ package cpu const cacheLineSize = 64 -func initOptions() {} +func initOptions() { + options = []option{ + {Name: "fastmisaligned", Feature: &RISCV64.HasFastMisaligned}, + {Name: "c", Feature: &RISCV64.HasC}, + {Name: "v", Feature: &RISCV64.HasV}, + {Name: "zba", Feature: &RISCV64.HasZba}, + {Name: "zbb", Feature: &RISCV64.HasZbb}, + {Name: "zbs", Feature: &RISCV64.HasZbs}, + } +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_x86.go index c29f5e4c..600a6807 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_x86.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -92,10 +92,8 @@ func archInit() { osSupportsAVX = isSet(1, eax) && isSet(2, eax) if runtime.GOOS == "darwin" { - // Darwin doesn't save/restore AVX-512 mask registers correctly across signal handlers. - // Since users can't rely on mask register contents, let's not advertise AVX-512 support. - // See issue 49233. - osSupportsAVX512 = false + // Darwin requires special AVX512 checks, see cpu_darwin_x86.go + osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512() } else { // Check if OPMASK and ZMM registers have OS support. osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax) diff --git a/src/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go b/src/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go new file mode 100644 index 00000000..4d0888b0 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go @@ -0,0 +1,98 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Minimal copy of x/sys/unix so the cpu package can make a +// system call on Darwin without depending on x/sys/unix. + +//go:build darwin && amd64 && gc + +package cpu + +import ( + "syscall" + "unsafe" +) + +type _C_int int32 + +// adapted from unix.Uname() at x/sys/unix/syscall_darwin.go L419 +func darwinOSRelease(release *[256]byte) error { + // from x/sys/unix/zerrors_openbsd_amd64.go + const ( + CTL_KERN = 0x1 + KERN_OSRELEASE = 0x2 + ) + + mib := []_C_int{CTL_KERN, KERN_OSRELEASE} + n := unsafe.Sizeof(*release) + + return sysctl(mib, &release[0], &n, nil, 0) +} + +type Errno = syscall.Errno + +var _zero uintptr // Single-word zero for use when we need a valid pointer to 0 bytes. + +// from x/sys/unix/zsyscall_darwin_amd64.go L791-807 +func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { + var _p0 unsafe.Pointer + if len(mib) > 0 { + _p0 = unsafe.Pointer(&mib[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + if _, _, err := syscall_syscall6( + libc_sysctl_trampoline_addr, + uintptr(_p0), + uintptr(len(mib)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen), + ); err != 0 { + return err + } + + return nil +} + +var libc_sysctl_trampoline_addr uintptr + +// adapted from internal/cpu/cpu_arm64_darwin.go +func darwinSysctlEnabled(name []byte) bool { + out := int32(0) + nout := unsafe.Sizeof(out) + if ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0); ret != nil { + return false + } + return out > 0 +} + +//go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib" + +var libc_sysctlbyname_trampoline_addr uintptr + +// adapted from runtime/sys_darwin.go in the pattern of sysctl() above, as defined in x/sys/unix +func sysctlbyname(name *byte, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { + if _, _, err := syscall_syscall6( + libc_sysctlbyname_trampoline_addr, + uintptr(unsafe.Pointer(name)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen), + 0, + ); err != 0 { + return err + } + + return nil +} + +//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib" + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) + +//go:linkname syscall_syscall6 syscall.syscall6 diff --git a/src/vendor/golang.org/x/text/LICENSE b/src/vendor/golang.org/x/text/LICENSE index 6a66aea5..2a7cf70d 100644 --- a/src/vendor/golang.org/x/text/LICENSE +++ b/src/vendor/golang.org/x/text/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/vendor/modules.txt b/src/vendor/modules.txt index b8a0b84a..d42f50b4 100644 --- a/src/vendor/modules.txt +++ b/src/vendor/modules.txt @@ -1,14 +1,12 @@ -# golang.org/x/crypto v0.23.1-0.20240603234054-0b431c7de36a -## explicit; go 1.18 +# golang.org/x/crypto v0.30.0 +## explicit; go 1.20 golang.org/x/crypto/chacha20 golang.org/x/crypto/chacha20poly1305 golang.org/x/crypto/cryptobyte golang.org/x/crypto/cryptobyte/asn1 -golang.org/x/crypto/hkdf golang.org/x/crypto/internal/alias golang.org/x/crypto/internal/poly1305 -golang.org/x/crypto/sha3 -# golang.org/x/net v0.25.1-0.20240603202750-6249541f2a6c +# golang.org/x/net v0.32.1-0.20250121202134-9a960c88dd98 ## explicit; go 1.18 golang.org/x/net/dns/dnsmessage golang.org/x/net/http/httpguts @@ -18,10 +16,10 @@ golang.org/x/net/idna golang.org/x/net/lif golang.org/x/net/nettest golang.org/x/net/route -# golang.org/x/sys v0.22.0 +# golang.org/x/sys v0.28.0 ## explicit; go 1.18 golang.org/x/sys/cpu -# golang.org/x/text v0.16.0 +# golang.org/x/text v0.21.0 ## explicit; go 1.18 golang.org/x/text/secure/bidirule golang.org/x/text/transform diff --git a/src/weak/doc.go b/src/weak/doc.go new file mode 100644 index 00000000..1af8e4c6 --- /dev/null +++ b/src/weak/doc.go @@ -0,0 +1,9 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package weak provides ways to safely reference memory weakly, +that is, without preventing its reclamation. +*/ +package weak diff --git a/src/weak/pointer.go b/src/weak/pointer.go new file mode 100644 index 00000000..e9d74208 --- /dev/null +++ b/src/weak/pointer.go @@ -0,0 +1,96 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package weak + +import ( + "internal/abi" + "runtime" + "unsafe" +) + +// Pointer is a weak pointer to a value of type T. +// +// Just like regular pointers, Pointer may reference any part of an +// object, such as a field of a struct or an element of an array. +// Objects that are only pointed to by weak pointers are not considered +// reachable, and once the object becomes unreachable, [Pointer.Value] +// may return nil. +// +// The primary use-cases for weak pointers are for implementing caches, +// canonicalization maps (like the unique package), and for tying together +// the lifetimes of separate values (for example, through a map with weak +// keys). +// +// Two Pointer values always compare equal if the pointers from which they were +// created compare equal. This property is retained even after the +// object referenced by the pointer used to create a weak reference is +// reclaimed. +// If multiple weak pointers are made to different offsets within the same object +// (for example, pointers to different fields of the same struct), those pointers +// will not compare equal. +// If a weak pointer is created from an object that becomes unreachable, but is +// then resurrected due to a finalizer, that weak pointer will not compare equal +// with weak pointers created after the resurrection. +// +// Calling [Make] with a nil pointer returns a weak pointer whose [Pointer.Value] +// always returns nil. The zero value of a Pointer behaves as if it were created +// by passing nil to [Make] and compares equal with such pointers. +// +// [Pointer.Value] is not guaranteed to eventually return nil. +// [Pointer.Value] may return nil as soon as the object becomes +// unreachable. +// Values stored in global variables, or that can be found by tracing +// pointers from a global variable, are reachable. A function argument or +// receiver may become unreachable at the last point where the function +// mentions it. To ensure [Pointer.Value] does not return nil, +// pass a pointer to the object to the [runtime.KeepAlive] function after +// the last point where the object must remain reachable. +// +// Note that because [Pointer.Value] is not guaranteed to eventually return +// nil, even after an object is no longer referenced, the runtime is allowed to +// perform a space-saving optimization that batches objects together in a single +// allocation slot. The weak pointer for an unreferenced object in such an +// allocation may never become nil if it always exists in the same batch as a +// referenced object. Typically, this batching only happens for tiny +// (on the order of 16 bytes or less) and pointer-free objects. +type Pointer[T any] struct { + // Mention T in the type definition to prevent conversions + // between Pointer types, like we do for sync/atomic.Pointer. + _ [0]*T + u unsafe.Pointer +} + +// Make creates a weak pointer from a pointer to some value of type T. +func Make[T any](ptr *T) Pointer[T] { + // Explicitly force ptr to escape to the heap. + ptr = abi.Escape(ptr) + + var u unsafe.Pointer + if ptr != nil { + u = runtime_registerWeakPointer(unsafe.Pointer(ptr)) + } + runtime.KeepAlive(ptr) + return Pointer[T]{u: u} +} + +// Value returns the original pointer used to create the weak pointer. +// It returns nil if the value pointed to by the original pointer was reclaimed by +// the garbage collector. +// If a weak pointer points to an object with a finalizer, then Value will +// return nil as soon as the object's finalizer is queued for execution. +func (p Pointer[T]) Value() *T { + if p.u == nil { + return nil + } + return (*T)(runtime_makeStrongFromWeak(p.u)) +} + +// Implemented in runtime. + +//go:linkname runtime_registerWeakPointer +func runtime_registerWeakPointer(unsafe.Pointer) unsafe.Pointer + +//go:linkname runtime_makeStrongFromWeak +func runtime_makeStrongFromWeak(unsafe.Pointer) unsafe.Pointer diff --git a/src/internal/weak/pointer_test.go b/src/weak/pointer_test.go similarity index 77% rename from src/internal/weak/pointer_test.go rename to src/weak/pointer_test.go index 5a861bb9..70c74338 100644 --- a/src/internal/weak/pointer_test.go +++ b/src/weak/pointer_test.go @@ -6,11 +6,13 @@ package weak_test import ( "context" - "internal/weak" + "internal/goarch" "runtime" "sync" "testing" "time" + "unsafe" + "weak" ) type T struct { @@ -21,40 +23,60 @@ type T struct { } func TestPointer(t *testing.T) { + var zero weak.Pointer[T] + if zero.Value() != nil { + t.Error("Value of zero value of weak.Pointer is not nil") + } + zeroNil := weak.Make[T](nil) + if zeroNil.Value() != nil { + t.Error("Value of weak.Make[T](nil) is not nil") + } + bt := new(T) wt := weak.Make(bt) - if st := wt.Strong(); st != bt { + if st := wt.Value(); st != bt { t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt) } // bt is still referenced. runtime.GC() - if st := wt.Strong(); st != bt { + if st := wt.Value(); st != bt { t.Fatalf("weak pointer is not the same as strong pointer after GC: %p vs. %p", st, bt) } // bt is no longer referenced. runtime.GC() - if st := wt.Strong(); st != nil { + if st := wt.Value(); st != nil { t.Fatalf("expected weak pointer to be nil, got %p", st) } } func TestPointerEquality(t *testing.T) { + var zero weak.Pointer[T] + zeroNil := weak.Make[T](nil) + if zero != zeroNil { + t.Error("weak.Make[T](nil) != zero value of weak.Pointer[T]") + } + bt := make([]*T, 10) wt := make([]weak.Pointer[T], 10) + wo := make([]weak.Pointer[int], 10) for i := range bt { bt[i] = new(T) wt[i] = weak.Make(bt[i]) + wo[i] = weak.Make(&bt[i].a) } for i := range bt { - st := wt[i].Strong() + st := wt[i].Value() if st != bt[i] { t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt[i]) } if wp := weak.Make(st); wp != wt[i] { t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wt[i]) } + if wp := weak.Make(&st.a); wp != wo[i] { + t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wo[i]) + } if i == 0 { continue } @@ -65,13 +87,16 @@ func TestPointerEquality(t *testing.T) { // bt is still referenced. runtime.GC() for i := range bt { - st := wt[i].Strong() + st := wt[i].Value() if st != bt[i] { t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt[i]) } if wp := weak.Make(st); wp != wt[i] { t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wt[i]) } + if wp := weak.Make(&st.a); wp != wo[i] { + t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wo[i]) + } if i == 0 { continue } @@ -83,7 +108,7 @@ func TestPointerEquality(t *testing.T) { // bt is no longer referenced. runtime.GC() for i := range bt { - st := wt[i].Strong() + st := wt[i].Value() if st != nil { t.Fatalf("expected weak pointer to be nil, got %p", st) } @@ -101,7 +126,7 @@ func TestPointerFinalizer(t *testing.T) { wt := weak.Make(bt) done := make(chan struct{}, 1) runtime.SetFinalizer(bt, func(bt *T) { - if wt.Strong() != nil { + if wt.Value() != nil { t.Errorf("weak pointer did not go nil before finalizer ran") } done <- struct{}{} @@ -109,7 +134,7 @@ func TestPointerFinalizer(t *testing.T) { // Make sure the weak pointer stays around while bt is live. runtime.GC() - if wt.Strong() == nil { + if wt.Value() == nil { t.Errorf("weak pointer went nil too soon") } runtime.KeepAlive(bt) @@ -118,7 +143,7 @@ func TestPointerFinalizer(t *testing.T) { // // Run one cycle to queue the finalizer. runtime.GC() - if wt.Strong() != nil { + if wt.Value() != nil { t.Errorf("weak pointer did not go nil when finalizer was enqueued") } @@ -127,11 +152,19 @@ func TestPointerFinalizer(t *testing.T) { // The weak pointer should still be nil after the finalizer runs. runtime.GC() - if wt.Strong() != nil { + if wt.Value() != nil { t.Errorf("weak pointer is non-nil even after finalization: %v", wt) } } +func TestPointerSize(t *testing.T) { + var p weak.Pointer[T] + size := unsafe.Sizeof(p) + if size != goarch.PtrSize { + t.Errorf("weak.Pointer[T] size = %d, want %d", size, goarch.PtrSize) + } +} + // Regression test for issue 69210. // // Weak-to-strong conversions must shade the new strong pointer, otherwise @@ -150,7 +183,7 @@ func TestIssue69210(t *testing.T) { // bug happens. Specifically, we want: // // 1. To create a whole bunch of objects that are only weakly-pointed-to, - // 2. To call Strong while the GC is in the mark phase, + // 2. To call Value while the GC is in the mark phase, // 3. The new strong pointer to be missed by the GC, // 4. The following GC cycle to mark a free object. // @@ -192,7 +225,7 @@ func TestIssue69210(t *testing.T) { wt := weak.Make(bt) bt = nil time.Sleep(1 * time.Millisecond) - bt = wt.Strong() + bt = wt.Value() if bt != nil { time.Sleep(4 * time.Millisecond) bt.t = bt @@ -210,3 +243,12 @@ func TestIssue69210(t *testing.T) { } wg.Wait() } + +func TestIssue70739(t *testing.T) { + x := make([]*int, 4<<16) + wx1 := weak.Make(&x[1<<16]) + wx2 := weak.Make(&x[1<<16]) + if wx1 != wx2 { + t.Fatal("failed to look up special and made duplicate weak handle; see issue #70739") + } +} diff --git a/test/alias2.go b/test/alias2.go index 95eb25a9..7a926cc4 100644 --- a/test/alias2.go +++ b/test/alias2.go @@ -46,8 +46,8 @@ var _ A0 = T0{} var _ T0 = A0{} // But aliases and original types cannot be used with new types based on them. -var _ N0 = T0{} // ERROR "cannot use T0{} \(value of type T0\) as N0 value in variable declaration" -var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type A0\) as N0 value in variable declaration" +var _ N0 = T0{} // ERROR "cannot use T0{} \(value of struct type T0\) as N0 value in variable declaration" +var _ N0 = A0{} // ERROR "cannot use A0{} \(value of struct type A0\) as N0 value in variable declaration" var _ A5 = Value{} @@ -82,10 +82,10 @@ func _() { var _ A0 = T0{} var _ T0 = A0{} - var _ N0 = T0{} // ERROR "cannot use T0{} \(value of type T0\) as N0 value in variable declaration" - var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type A0\) as N0 value in variable declaration" + var _ N0 = T0{} // ERROR "cannot use T0{} \(value of struct type T0\) as N0 value in variable declaration" + var _ N0 = A0{} // ERROR "cannot use A0{} \(value of struct type A0\) as N0 value in variable declaration" - var _ A5 = Value{} // ERROR "cannot use Value{} \(value of type reflect\.Value\) as A5 value in variable declaration" + var _ A5 = Value{} // ERROR "cannot use Value{} \(value of struct type reflect\.Value\) as A5 value in variable declaration" } // Invalid type alias declarations. diff --git a/test/assign.go b/test/assign.go index bdec58b7..5beffee6 100644 --- a/test/assign.go +++ b/test/assign.go @@ -9,7 +9,10 @@ package main -import "sync" +import ( + "sync" + "time" +) type T struct { int @@ -38,7 +41,7 @@ func main() { _ = x } { - x := sync.Mutex{0, 0} // ERROR "assignment.*Mutex" + x := time.Time{0, 0, nil} // ERROR "assignment.*Time" _ = x } { diff --git a/test/checkbce.go b/test/checkbce.go index 71acfb71..49d04744 100644 --- a/test/checkbce.go +++ b/test/checkbce.go @@ -159,16 +159,14 @@ func decode1(data []byte) (x uint64) { } func decode2(data []byte) (x uint64) { - // TODO(rasky): this should behave like decode1 and compile to no - // boundchecks. We're currently not able to remove all of them. for len(data) >= 32 { x += binary.BigEndian.Uint64(data) data = data[8:] - x += binary.BigEndian.Uint64(data) // ERROR "Found IsInBounds$" + x += binary.BigEndian.Uint64(data) data = data[8:] - x += binary.BigEndian.Uint64(data) // ERROR "Found IsInBounds$" + x += binary.BigEndian.Uint64(data) data = data[8:] - x += binary.BigEndian.Uint64(data) // ERROR "Found IsInBounds$" + x += binary.BigEndian.Uint64(data) data = data[8:] } return x diff --git a/test/closure3.dir/main.go b/test/closure3.dir/main.go index 441da701..7018a473 100644 --- a/test/closure3.dir/main.go +++ b/test/closure3.dir/main.go @@ -20,6 +20,7 @@ func main() { if x := func() int { // ERROR "can inline main.func2" "func literal does not escape" return 1 }; x() != 1 { // ERROR "inlining call to main.func2" + _ = x // prevent simple deadcode elimination after inlining ppanic("x() != 1") } } @@ -33,6 +34,7 @@ func main() { if y := func(x int) int { // ERROR "can inline main.func4" "func literal does not escape" return x + 2 }; y(40) != 42 { // ERROR "inlining call to main.func4" + _ = y // prevent simple deadcode elimination after inlining ppanic("y(40) != 42") } } @@ -50,7 +52,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func7" y := func(x int) int { // ERROR "can inline main.func7.1" "func literal does not escape" return x + 2 } @@ -60,7 +62,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func7" } { @@ -76,7 +78,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func10" y := func(x int) int { // ERROR "can inline main.func10.1" "func literal does not escape" return x + 2 } @@ -86,7 +88,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func10" } { @@ -94,7 +96,7 @@ func main() { return x + 2 } y, sink = func() (func(int) int, int) { // ERROR "can inline main.func12" - return func(x int) int { // ERROR "can inline main.func12" + return func(x int) int { // ERROR "can inline main.func12" "func literal escapes to heap" return x + 1 }, 42 }() // ERROR "func literal does not escape" "inlining call to main.func12" @@ -104,19 +106,19 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func13" y := func(x int) int { // ERROR "func literal does not escape" "can inline main.func13.1" return x + 2 } - y, sink = func() (func(int) int, int) { // ERROR "can inline main.func13.2" - return func(x int) int { // ERROR "can inline main.func13.2" + y, sink = func() (func(int) int, int) { // ERROR "can inline main.func13.2" "can inline main.main.func13.func35" + return func(x int) int { // ERROR "can inline main.func13.2" "func literal escapes to heap" return x + 1 }, 42 }() // ERROR "func literal does not escape" "inlining call to main.func13.2" if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func13" "inlining call to main.main.func13.func35" } { @@ -132,7 +134,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func16" y := func(x int) int { // ERROR "can inline main.func16.1" "func literal does not escape" return x + 2 } @@ -142,7 +144,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func16" "map\[int\]func\(int\) int{...} does not escape" "func literal escapes to heap" } { @@ -158,7 +160,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func19" y := func(x int) int { // ERROR "can inline main.func19.1" "func literal does not escape" return x + 2 } @@ -168,7 +170,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func19" } { @@ -181,6 +183,7 @@ func main() { if y := func() int { // ERROR "can inline main.func21" "func literal does not escape" return x }; y() != 42 { // ERROR "inlining call to main.func21" + _ = y // prevent simple deadcode elimination after inlining ppanic("y() != 42") } } @@ -188,17 +191,18 @@ func main() { { x := 42 if z := func(y int) int { // ERROR "can inline main.func22" - return func() int { // ERROR "can inline main.func22.1" "can inline main.main.func22.func30" + return func() int { // ERROR "can inline main.func22.1" "can inline main.main.func22.func40" return x + y }() // ERROR "inlining call to main.func22.1" - }(1); z != 43 { // ERROR "inlining call to main.func22" "inlining call to main.main.func22.func30" + }(1); z != 43 { // ERROR "inlining call to main.func22" "inlining call to main.main.func22.func40" ppanic("z != 43") } if z := func(y int) int { // ERROR "func literal does not escape" "can inline main.func23" - return func() int { // ERROR "can inline main.func23.1" "can inline main.main.func23.func31" + return func() int { // ERROR "can inline main.func23.1" "can inline main.main.func23.func41" return x + y }() // ERROR "inlining call to main.func23.1" - }; z(1) != 43 { // ERROR "inlining call to main.func23" "inlining call to main.main.func23.func31" + }; z(1) != 43 { // ERROR "inlining call to main.func23" "inlining call to main.main.func23.func41" + _ = z // prevent simple deadcode elimination after inlining ppanic("z(1) != 43") } } @@ -206,10 +210,10 @@ func main() { { a := 1 func() { // ERROR "can inline main.func24" - func() { // ERROR "can inline main.func24" "can inline main.main.func24.func32" + func() { // ERROR "can inline main.func24" "can inline main.main.func24.func42" a = 2 }() // ERROR "inlining call to main.func24" - }() // ERROR "inlining call to main.func24" "inlining call to main.main.func24.func32" + }() // ERROR "inlining call to main.func24" "inlining call to main.main.func24.func42" if a != 2 { ppanic("a != 2") } @@ -217,14 +221,14 @@ func main() { { b := 2 - func(b int) { // ERROR "func literal does not escape" - func() { // ERROR "can inline main.func25.1" + func(b int) { // ERROR "can inline main.func25" + func() { // ERROR "can inline main.func25.1" "can inline main.main.func25.func43" b = 3 }() // ERROR "inlining call to main.func25.1" if b != 3 { ppanic("b != 3") } - }(b) + }(b) // ERROR "inlining call to main.func25" "inlining call to main.main.func25.func43" if b != 2 { ppanic("b != 2") } @@ -254,13 +258,13 @@ func main() { // revisit those. E.g., func34 and func36 are constructed by the inliner. if r := func(x int) int { // ERROR "can inline main.func27" b := 3 - return func(y int) int { // ERROR "can inline main.func27.1" "can inline main.main.func27.func34" + return func(y int) int { // ERROR "can inline main.func27.1" "can inline main.main.func27.func45" c := 5 - return func(z int) int { // ERROR "can inline main.func27.1.1" "can inline main.main.func27.func34.1" "can inline main.func27.main.func27.1.2" "can inline main.main.func27.main.main.func27.func34.func36" + return func(z int) int { // ERROR "can inline main.func27.1.1" "can inline main.main.func27.func45.1" "can inline main.func27.main.func27.1.2" "can inline main.main.func27.main.main.func27.func45.func48" return a*x + b*y + c*z }(10) // ERROR "inlining call to main.func27.1.1" }(100) // ERROR "inlining call to main.func27.1" "inlining call to main.func27.main.func27.1.2" - }(1000); r != 2350 { // ERROR "inlining call to main.func27" "inlining call to main.main.func27.func34" "inlining call to main.main.func27.main.main.func27.func34.func36" + }(1000); r != 2350 { // ERROR "inlining call to main.func27" "inlining call to main.main.func27.func45" "inlining call to main.main.func27.main.main.func27.func45.func48" ppanic("r != 2350") } } @@ -269,16 +273,16 @@ func main() { a := 2 if r := func(x int) int { // ERROR "can inline main.func28" b := 3 - return func(y int) int { // ERROR "can inline main.func28.1" "can inline main.main.func28.func35" + return func(y int) int { // ERROR "can inline main.func28.1" "can inline main.main.func28.func46" c := 5 - func(z int) { // ERROR "can inline main.func28.1.1" "can inline main.func28.main.func28.1.2" "can inline main.main.func28.func35.1" "can inline main.main.func28.main.main.func28.func35.func37" + func(z int) { // ERROR "can inline main.func28.1.1" "can inline main.func28.main.func28.1.2" "can inline main.main.func28.func46.1" "can inline main.main.func28.main.main.func28.func46.func49" a = a * x b = b * y c = c * z }(10) // ERROR "inlining call to main.func28.1.1" return a + c }(100) + b // ERROR "inlining call to main.func28.1" "inlining call to main.func28.main.func28.1.2" - }(1000); r != 2350 { // ERROR "inlining call to main.func28" "inlining call to main.main.func28.func35" "inlining call to main.main.func28.main.main.func28.func35.func37" + }(1000); r != 2350 { // ERROR "inlining call to main.func28" "inlining call to main.main.func28.func46" "inlining call to main.main.func28.main.main.func28.func46.func49" ppanic("r != 2350") } if a != 2000 { @@ -287,6 +291,25 @@ func main() { } } +//go:noinline +func notmain() { + { + // This duplicates the first block in main, but without the "_ = x" for closure x. + // This allows dead code elimination of x before escape analysis, + // thus "func literal does not escape" should not appear. + if x := func() int { // ERROR "can inline notmain.func1" + return 1 + }(); x != 1 { // ERROR "inlining call to notmain.func1" + ppanic("x != 1") + } + if x := func() int { // ERROR "can inline notmain.func2" + return 1 + }; x() != 1 { // ERROR "inlining call to notmain.func2" + ppanic("x() != 1") + } + } +} + //go:noinline func ppanic(s string) { // ERROR "leaking param: s" panic(s) // ERROR "s escapes to heap" diff --git a/test/codegen/arithmetic.go b/test/codegen/arithmetic.go index e474a10b..4b47f6c1 100644 --- a/test/codegen/arithmetic.go +++ b/test/codegen/arithmetic.go @@ -629,3 +629,39 @@ func constantFold3(i, j int) int { r := (5 * i) * (6 * j) return r } + +// ----------------- // +// Integer Min/Max // +// ----------------- // + +func Int64Min(a, b int64) int64 { + // amd64: "CMPQ","CMOVQLT" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLT\t" + // riscv64/rva22u64:"MIN\t" + return min(a, b) +} + +func Int64Max(a, b int64) int64 { + // amd64: "CMPQ","CMOVQGT" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLT\t" + // riscv64/rva22u64:"MAX\t" + return max(a, b) +} + +func Uint64Min(a, b uint64) uint64 { + // amd64: "CMPQ","CMOVQCS" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLTU" + // riscv64/rva22u64:"MINU" + return min(a, b) +} + +func Uint64Max(a, b uint64) uint64 { + // amd64: "CMPQ","CMOVQHI" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLTU" + // riscv64/rva22u64:"MAXU" + return max(a, b) +} diff --git a/test/codegen/atomics.go b/test/codegen/atomics.go index feaa31b9..14024dcd 100644 --- a/test/codegen/atomics.go +++ b/test/codegen/atomics.go @@ -22,6 +22,74 @@ func (c *Counter) Increment() { // arm64/v8.1:"LDADDALW" // arm64/v8.0:".*arm64HasATOMICS" // arm64/v8.1:-".*arm64HasATOMICS" + // amd64:"LOCK",-"CMPXCHG" atomic.AddInt32(&c.count, 1) } +func atomicLogical64(x *atomic.Uint64) uint64 { + var r uint64 + + // arm64/v8.0:"LDCLRALD" + // arm64/v8.1:"LDCLRALD" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // On amd64, make sure we use LOCK+AND instead of CMPXCHG when we don't use the result. + // amd64:"LOCK",-"CMPXCHGQ" + x.And(11) + // arm64/v8.0:"LDCLRALD" + // arm64/v8.1:"LDCLRALD" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // amd64:"LOCK","CMPXCHGQ" + r += x.And(22) + + // arm64/v8.0:"LDORALD" + // arm64/v8.1:"LDORALD" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // On amd64, make sure we use LOCK+OR instead of CMPXCHG when we don't use the result. + // amd64:"LOCK",-"CMPXCHGQ" + x.Or(33) + // arm64/v8.0:"LDORALD" + // arm64/v8.1:"LDORALD" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // amd64:"LOCK","CMPXCHGQ" + r += x.Or(44) + + return r +} + +func atomicLogical32(x *atomic.Uint32) uint32 { + var r uint32 + + // arm64/v8.0:"LDCLRALW" + // arm64/v8.1:"LDCLRALW" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // On amd64, make sure we use LOCK+AND instead of CMPXCHG when we don't use the result. + // amd64:"LOCK",-"CMPXCHGL" + x.And(11) + // arm64/v8.0:"LDCLRALW" + // arm64/v8.1:"LDCLRALW" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // amd64:"LOCK","CMPXCHGL" + r += x.And(22) + + // arm64/v8.0:"LDORALW" + // arm64/v8.1:"LDORALW" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // On amd64, make sure we use LOCK+OR instead of CMPXCHG when we don't use the result. + // amd64:"LOCK",-"CMPXCHGL" + x.Or(33) + // arm64/v8.0:"LDORALW" + // arm64/v8.1:"LDORALW" + // arm64/v8.0:".*arm64HasATOMICS" + // arm64/v8.1:-".*arm64HasATOMICS" + // amd64:"LOCK","CMPXCHGL" + r += x.Or(44) + + return r +} diff --git a/test/codegen/bitfield.go b/test/codegen/bitfield.go index 3276af3d..44cf09f8 100644 --- a/test/codegen/bitfield.go +++ b/test/codegen/bitfield.go @@ -358,11 +358,15 @@ func rev16w(c uint32) (uint32, uint32, uint32) { func shift(x uint32, y uint16, z uint8) uint64 { // arm64:-`MOVWU`,-`LSR\t[$]32` + // loong64:-`MOVWU`,-`SRLV\t[$]32` a := uint64(x) >> 32 // arm64:-`MOVHU + // loong64:-`MOVHU`,-`SRLV\t[$]16` b := uint64(y) >> 16 // arm64:-`MOVBU` + // loong64:-`MOVBU`,-`SRLV\t[$]8` c := uint64(z) >> 8 // arm64:`MOVD\tZR`,-`ADD\tR[0-9]+>>16`,-`ADD\tR[0-9]+>>8`, + // loong64:`MOVV\t[$]0`,-`ADDVU` return a + b + c } diff --git a/test/codegen/bits.go b/test/codegen/bits.go index 554e363e..354dbf40 100644 --- a/test/codegen/bits.go +++ b/test/codegen/bits.go @@ -366,6 +366,7 @@ func issue48467(x, y uint64) uint64 { func foldConst(x, y uint64) uint64 { // arm64: "ADDS\t[$]7",-"MOVD\t[$]7" + // ppc64x: "ADDC\t[$]7," d, b := bits.Add64(x, 7, 0) return b & d } diff --git a/test/codegen/bool.go b/test/codegen/bool.go index 990a9ed1..164ca1b2 100644 --- a/test/codegen/bool.go +++ b/test/codegen/bool.go @@ -217,53 +217,53 @@ func TestSetInvGeFp64(x float64, y float64) bool { } func TestLogicalCompareZero(x *[64]uint64) { // ppc64x:"ANDCC",^"AND" - b := x[0]&3 - if b!=0 { + b := x[0] & 3 + if b != 0 { x[0] = b } // ppc64x:"ANDCC",^"AND" - b = x[1]&x[2] - if b!=0 { + b = x[1] & x[2] + if b != 0 { x[1] = b } // ppc64x:"ANDNCC",^"ANDN" - b = x[1]&^x[2] - if b!=0 { + b = x[1] &^ x[2] + if b != 0 { x[1] = b } // ppc64x:"ORCC",^"OR" - b = x[3]|x[4] - if b!=0 { + b = x[3] | x[4] + if b != 0 { x[3] = b } // ppc64x:"SUBCC",^"SUB" - b = x[5]-x[6] - if b!=0 { + b = x[5] - x[6] + if b != 0 { x[5] = b } // ppc64x:"NORCC",^"NOR" - b = ^(x[5]|x[6]) - if b!=0 { + b = ^(x[5] | x[6]) + if b != 0 { x[5] = b } // ppc64x:"XORCC",^"XOR" - b = x[7]^x[8] - if b!=0 { + b = x[7] ^ x[8] + if b != 0 { x[7] = b } // ppc64x:"ADDCC",^"ADD" - b = x[9]+x[10] - if b!=0 { + b = x[9] + x[10] + if b != 0 { x[9] = b } // ppc64x:"NEGCC",^"NEG" b = -x[11] - if b!=0 { + if b != 0 { x[11] = b } // ppc64x:"CNTLZDCC",^"CNTLZD" b = uint64(bits.LeadingZeros64(x[12])) - if b!=0 { + if b != 0 { x[12] = b } @@ -273,4 +273,17 @@ func TestLogicalCompareZero(x *[64]uint64) { x[12] = uint64(c) } + // ppc64x:"MULHDUCC",^"MULHDU" + hi, _ := bits.Mul64(x[13], x[14]) + if hi != 0 { + x[14] = hi + } + +} + +func constantWrite(b bool, p *bool) { + if b { + // amd64:`MOVB\t[$]1, \(` + *p = b + } } diff --git a/test/codegen/comparisons.go b/test/codegen/comparisons.go index e585045a..5fbb31c0 100644 --- a/test/codegen/comparisons.go +++ b/test/codegen/comparisons.go @@ -233,9 +233,9 @@ func CmpToZero(a, b, d int32, e, f int64, deOptC0, deOptC1 bool) int32 { // arm:`AND`,-`TST` // 386:`ANDL` c6 := a&d >= 0 - // arm64:`TST\sR[0-9]+<<3,\sR[0-9]+` + // For arm64, could be TST+BGE or AND+TBZ c7 := e&(f<<3) < 0 - // arm64:`CMN\sR[0-9]+<<3,\sR[0-9]+` + // For arm64, could be CMN+BPL or ADD+TBZ c8 := e+(f<<3) < 0 // arm64:`TST\sR[0-9],\sR[0-9]+` c9 := e&(-19) < 0 @@ -268,7 +268,7 @@ func CmpToZero(a, b, d int32, e, f int64, deOptC0, deOptC1 bool) int32 { } } -func CmpLogicalToZero(a, b, c uint32, d, e uint64) uint64 { +func CmpLogicalToZero(a, b, c uint32, d, e, f, g uint64) uint64 { // ppc64x:"ANDCC",-"CMPW" // wasm:"I64Eqz",-"I32Eqz",-"I64ExtendI32U",-"I32WrapI64" @@ -289,7 +289,7 @@ func CmpLogicalToZero(a, b, c uint32, d, e uint64) uint64 { } // ppc64x:"ORCC",-"CMP" // wasm:"I64Eqz",-"I32Eqz",-"I64ExtendI32U",-"I32WrapI64" - if d|e == 0 { + if f|g == 0 { return 1 } diff --git a/test/codegen/copy.go b/test/codegen/copy.go index 17ee8bc8..4329c6d7 100644 --- a/test/codegen/copy.go +++ b/test/codegen/copy.go @@ -95,6 +95,7 @@ func moveArchLowering1(b []byte, x *[1]byte) { _ = b[1] // amd64:-".*memmove" // arm64:-".*memmove" + // loong64:-".*memmove" // ppc64x:-".*memmove" copy(b, x[:]) } @@ -103,6 +104,7 @@ func moveArchLowering2(b []byte, x *[2]byte) { _ = b[2] // amd64:-".*memmove" // arm64:-".*memmove" + // loong64:-".*memmove" // ppc64x:-".*memmove" copy(b, x[:]) } @@ -111,6 +113,7 @@ func moveArchLowering4(b []byte, x *[4]byte) { _ = b[4] // amd64:-".*memmove" // arm64:-".*memmove" + // loong64:-".*memmove" // ppc64x:-".*memmove" copy(b, x[:]) } diff --git a/test/codegen/floats.go b/test/codegen/floats.go index baa745bd..1b85eba3 100644 --- a/test/codegen/floats.go +++ b/test/codegen/floats.go @@ -54,11 +54,13 @@ func DivPow2(f1, f2, f3 float64) (float64, float64, float64) { func indexLoad(b0 []float32, b1 float32, idx int) float32 { // arm64:`FMOVS\s\(R[0-9]+\)\(R[0-9]+<<2\),\sF[0-9]+` + // loong64:`MOVF\s\(R[0-9]+\)\(R[0-9]+\),\sF[0-9]+` return b0[idx] * b1 } func indexStore(b0 []float64, b1 float64, idx int) { // arm64:`FMOVD\sF[0-9]+,\s\(R[0-9]+\)\(R[0-9]+<<3\)` + // loong64:`MOVD\sF[0-9]+,\s\(R[0-9]+\)\(R[0-9]+\)` b0[idx] = b1 } @@ -70,6 +72,7 @@ func FusedAdd32(x, y, z float32) float32 { // s390x:"FMADDS\t" // ppc64x:"FMADDS\t" // arm64:"FMADDS" + // loong64:"FMADDF\t" // riscv64:"FMADDS\t" return x*y + z } @@ -78,11 +81,13 @@ func FusedSub32_a(x, y, z float32) float32 { // s390x:"FMSUBS\t" // ppc64x:"FMSUBS\t" // riscv64:"FMSUBS\t" + // loong64:"FMSUBF\t" return x*y - z } func FusedSub32_b(x, y, z float32) float32 { // arm64:"FMSUBS" + // loong64:"FNMSUBF\t" // riscv64:"FNMSUBS\t" return z - x*y } @@ -91,6 +96,7 @@ func FusedAdd64(x, y, z float64) float64 { // s390x:"FMADD\t" // ppc64x:"FMADD\t" // arm64:"FMADDD" + // loong64:"FMADDD\t" // riscv64:"FMADDD\t" return x*y + z } @@ -99,11 +105,13 @@ func FusedSub64_a(x, y, z float64) float64 { // s390x:"FMSUB\t" // ppc64x:"FMSUB\t" // riscv64:"FMSUBD\t" + // loong64:"FMSUBD\t" return x*y - z } func FusedSub64_b(x, y, z float64) float64 { // arm64:"FMSUBD" + // loong64:"FNMSUBD\t" // riscv64:"FNMSUBD\t" return z - x*y } @@ -164,6 +172,7 @@ func ArrayCopy(a [16]byte) (b [16]byte) { func Float64Min(a, b float64) float64 { // amd64:"MINSD" // arm64:"FMIND" + // loong64:"FMIND" // riscv64:"FMIN" // ppc64/power9:"XSMINJDP" // ppc64/power10:"XSMINJDP" @@ -173,6 +182,7 @@ func Float64Min(a, b float64) float64 { func Float64Max(a, b float64) float64 { // amd64:"MINSD" // arm64:"FMAXD" + // loong64:"FMAXD" // riscv64:"FMAX" // ppc64/power9:"XSMAXJDP" // ppc64/power10:"XSMAXJDP" @@ -182,6 +192,7 @@ func Float64Max(a, b float64) float64 { func Float32Min(a, b float32) float32 { // amd64:"MINSS" // arm64:"FMINS" + // loong64:"FMINF" // riscv64:"FMINS" // ppc64/power9:"XSMINJDP" // ppc64/power10:"XSMINJDP" @@ -191,6 +202,7 @@ func Float32Min(a, b float32) float32 { func Float32Max(a, b float32) float32 { // amd64:"MINSS" // arm64:"FMAXS" + // loong64:"FMAXF" // riscv64:"FMAXS" // ppc64/power9:"XSMAXJDP" // ppc64/power10:"XSMAXJDP" @@ -227,3 +239,12 @@ func Float64DenormalFloat32Constant() float64 { // ppc64x:"FMOVD\t[$]f64\\.3800000000000000\\(SB\\)" return 0x1p-127 } + +func Float64ConstantStore(p *float64) { + // amd64: "MOVQ\t[$]4617801906721357038" + *p = 5.432 +} +func Float32ConstantStore(p *float32) { + // amd64: "MOVL\t[$]1085133554" + *p = 5.432 +} diff --git a/src/cmd/vendor/golang.org/x/tools/cmd/bisect/go119.go b/test/codegen/issue59297.go similarity index 52% rename from src/cmd/vendor/golang.org/x/tools/cmd/bisect/go119.go rename to test/codegen/issue59297.go index debe4e0c..1703a1a8 100644 --- a/src/cmd/vendor/golang.org/x/tools/cmd/bisect/go119.go +++ b/test/codegen/issue59297.go @@ -1,13 +1,17 @@ +// asmcheck + // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !go1.20 +package codegen -package main - -import "os/exec" - -func cmdInterrupt(cmd *exec.Cmd) { - // cmd.Cancel and cmd.WaitDelay not available before Go 1.20. +func f(x, y int, p *int) { + // amd64:`MOVQ\sAX, BX` + h(8, x) + *p = y +} + +//go:noinline +func h(a, b int) { } diff --git a/test/codegen/issue68845.go b/test/codegen/issue68845.go new file mode 100644 index 00000000..3b22471d --- /dev/null +++ b/test/codegen/issue68845.go @@ -0,0 +1,52 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +type T1 struct { + x string +} + +func f1() *T1 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T1{} +} + +type T2 struct { + x, y string +} + +func f2() *T2 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T2{} +} + +type T3 struct { + x complex128 +} + +func f3() *T3 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T3{} +} + +type T4 struct { + x []byte +} + +func f4() *T4 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T4{} +} + +type T5 struct { + x any +} + +func f5() *T5 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T5{} +} diff --git a/test/codegen/issue69635.go b/test/codegen/issue69635.go new file mode 100644 index 00000000..de68bf12 --- /dev/null +++ b/test/codegen/issue69635.go @@ -0,0 +1,13 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func calc(a uint64) uint64 { + v := a >> 20 & 0x7f + // amd64: `SHRQ\s\$17, AX$`, `ANDL\s\$1016, AX$` + return v << 3 +} diff --git a/test/codegen/maps.go b/test/codegen/maps.go index 25505799..c4aed335 100644 --- a/test/codegen/maps.go +++ b/test/codegen/maps.go @@ -74,7 +74,7 @@ func LookupStringConversionKeyedArrayLit(m map[[2]string]int, bytes []byte) int func MapClearReflexive(m map[int]int) { // amd64:`.*runtime\.mapclear` - // amd64:-`.*runtime\.mapiterinit` + // amd64:-`.*runtime\.(mapiterinit|mapIterStart)` for k := range m { delete(m, k) } @@ -83,7 +83,7 @@ func MapClearReflexive(m map[int]int) { func MapClearIndirect(m map[int]int) { s := struct{ m map[int]int }{m: m} // amd64:`.*runtime\.mapclear` - // amd64:-`.*runtime\.mapiterinit` + // amd64:-`.*runtime\.(mapiterinit|mapIterStart)` for k := range s.m { delete(s.m, k) } @@ -91,14 +91,14 @@ func MapClearIndirect(m map[int]int) { func MapClearPointer(m map[*byte]int) { // amd64:`.*runtime\.mapclear` - // amd64:-`.*runtime\.mapiterinit` + // amd64:-`.*runtime\.(mapiterinit|mapIterStart)` for k := range m { delete(m, k) } } func MapClearNotReflexive(m map[float64]int) { - // amd64:`.*runtime\.mapiterinit` + // amd64:`.*runtime\.(mapiterinit|mapIterStart)` // amd64:-`.*runtime\.mapclear` for k := range m { delete(m, k) @@ -106,7 +106,7 @@ func MapClearNotReflexive(m map[float64]int) { } func MapClearInterface(m map[interface{}]int) { - // amd64:`.*runtime\.mapiterinit` + // amd64:`.*runtime\.(mapiterinit|mapIterStart)` // amd64:-`.*runtime\.mapclear` for k := range m { delete(m, k) @@ -115,7 +115,7 @@ func MapClearInterface(m map[interface{}]int) { func MapClearSideEffect(m map[int]int) int { k := 0 - // amd64:`.*runtime\.mapiterinit` + // amd64:`.*runtime\.(mapiterinit|mapIterStart)` // amd64:-`.*runtime\.mapclear` for k = range m { delete(m, k) diff --git a/test/codegen/math.go b/test/codegen/math.go index eb6e927d..4ce5fa41 100644 --- a/test/codegen/math.go +++ b/test/codegen/math.go @@ -132,6 +132,7 @@ func fma(x, y, z float64) float64 { // amd64:"VFMADD231SD" // arm/6:"FMULAD" // arm64:"FMADDD" + // loong64:"FMADDD" // s390x:"FMADD" // ppc64x:"FMADD" // riscv64:"FMADDD" @@ -156,6 +157,7 @@ func fnma(x, y, z float64) float64 { func fromFloat64(f64 float64) uint64 { // amd64:"MOVQ\tX.*, [^X].*" // arm64:"FMOVD\tF.*, R.*" + // loong64:"MOVV\tF.*, R.*" // ppc64x:"MFVSRD" // mips64/hardfloat:"MOVV\tF.*, R.*" return math.Float64bits(f64+1) + 1 @@ -164,6 +166,7 @@ func fromFloat64(f64 float64) uint64 { func fromFloat32(f32 float32) uint32 { // amd64:"MOVL\tX.*, [^X].*" // arm64:"FMOVS\tF.*, R.*" + // loong64:"MOVW\tF.*, R.*" // mips64/hardfloat:"MOVW\tF.*, R.*" return math.Float32bits(f32+1) + 1 } @@ -171,6 +174,7 @@ func fromFloat32(f32 float32) uint32 { func toFloat64(u64 uint64) float64 { // amd64:"MOVQ\t[^X].*, X.*" // arm64:"FMOVD\tR.*, F.*" + // loong64:"MOVV\tR.*, F.*" // ppc64x:"MTVSRD" // mips64/hardfloat:"MOVV\tR.*, F.*" return math.Float64frombits(u64+1) + 1 @@ -179,6 +183,7 @@ func toFloat64(u64 uint64) float64 { func toFloat32(u32 uint32) float32 { // amd64:"MOVL\t[^X].*, X.*" // arm64:"FMOVS\tR.*, F.*" + // loong64:"MOVW\tR.*, F.*" // mips64/hardfloat:"MOVW\tR.*, F.*" return math.Float32frombits(u32+1) + 1 } diff --git a/test/codegen/mathbits.go b/test/codegen/mathbits.go index a743eaf3..baed4f7c 100644 --- a/test/codegen/mathbits.go +++ b/test/codegen/mathbits.go @@ -17,6 +17,7 @@ func LeadingZeros(n uint) int { // amd64/v3:"LZCNTQ", -"BSRQ" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV",-"SUB" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"CNTLZD" @@ -28,6 +29,7 @@ func LeadingZeros64(n uint64) int { // amd64/v3:"LZCNTQ", -"BSRQ" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV",-"SUB" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"CNTLZD" @@ -39,6 +41,7 @@ func LeadingZeros32(n uint32) int { // amd64/v3: "LZCNTL",- "BSRL" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZW" + // loong64:"CLZW",-"SUB" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"CNTLZW" @@ -50,6 +53,7 @@ func LeadingZeros16(n uint16) int { // amd64/v3: "LZCNTL",- "BSRL" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"CNTLZD" @@ -61,6 +65,7 @@ func LeadingZeros8(n uint8) int { // amd64/v3: "LZCNTL",- "BSRL" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"CNTLZD" @@ -76,6 +81,7 @@ func Len(n uint) int { // amd64/v3: "LZCNTQ" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" @@ -87,6 +93,7 @@ func Len64(n uint64) int { // amd64/v3: "LZCNTQ" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" @@ -94,15 +101,22 @@ func Len64(n uint64) int { } func SubFromLen64(n uint64) int { + // loong64:"CLZV",-"ADD" // ppc64x:"CNTLZD",-"SUBC" return 64 - bits.Len64(n) } +func CompareWithLen64(n uint64) bool { + // loong64:"CLZV",-"ADD",-"[$]64",-"[$]9" + return bits.Len64(n) < 9 +} + func Len32(n uint32) int { // amd64/v1,amd64/v2:"BSRQ","LEAQ",-"CMOVQEQ" // amd64/v3: "LZCNTL" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZW" // mips:"CLZ" // wasm:"I64Clz" // ppc64x: "CNTLZW" @@ -114,6 +128,7 @@ func Len16(n uint16) int { // amd64/v3: "LZCNTL" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" @@ -125,6 +140,7 @@ func Len8(n uint8) int { // amd64/v3: "LZCNTL" // s390x:"FLOGR" // arm:"CLZ" arm64:"CLZ" + // loong64:"CLZV" // mips:"CLZ" // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" @@ -140,6 +156,7 @@ func OnesCount(n uint) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTQ" // arm64:"VCNT","VUADDLV" + // loong64:"VPCNTV" // s390x:"POPCNT" // ppc64x:"POPCNTD" // wasm:"I64Popcnt" @@ -150,6 +167,7 @@ func OnesCount64(n uint64) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTQ" // arm64:"VCNT","VUADDLV" + // loong64:"VPCNTV" // s390x:"POPCNT" // ppc64x:"POPCNTD" // wasm:"I64Popcnt" @@ -160,6 +178,7 @@ func OnesCount32(n uint32) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTL" // arm64:"VCNT","VUADDLV" + // loong64:"VPCNTW" // s390x:"POPCNT" // ppc64x:"POPCNTW" // wasm:"I64Popcnt" @@ -170,6 +189,7 @@ func OnesCount16(n uint16) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTL" // arm64:"VCNT","VUADDLV" + // loong64:"VPCNTH" // s390x:"POPCNT" // ppc64x:"POPCNTW" // wasm:"I64Popcnt" @@ -183,6 +203,35 @@ func OnesCount8(n uint8) int { return bits.OnesCount8(n) } +// ------------------ // +// bits.Reverse // +// ------------------ // + +func Reverse(n uint) uint { + // loong64:"BITREVV" + return bits.Reverse(n) +} + +func Reverse64(n uint64) uint64 { + // loong64:"BITREVV" + return bits.Reverse64(n) +} + +func Reverse32(n uint32) uint32 { + // loong64:"BITREVW" + return bits.Reverse32(n) +} + +func Reverse16(n uint16) uint16 { + // loong64:"BITREV4B","REVB2H" + return bits.Reverse16(n) +} + +func Reverse8(n uint8) uint8 { + // loong64:"BITREV4B" + return bits.Reverse8(n) +} + // ----------------------- // // bits.ReverseBytes // // ----------------------- // @@ -192,6 +241,7 @@ func ReverseBytes(n uint) uint { // 386:"BSWAPL" // s390x:"MOVDBR" // arm64:"REV" + // loong64:"REVBV" return bits.ReverseBytes(n) } @@ -201,6 +251,7 @@ func ReverseBytes64(n uint64) uint64 { // s390x:"MOVDBR" // arm64:"REV" // ppc64x/power10: "BRD" + // loong64:"REVBV" return bits.ReverseBytes64(n) } @@ -209,6 +260,7 @@ func ReverseBytes32(n uint32) uint32 { // 386:"BSWAPL" // s390x:"MOVWBR" // arm64:"REVW" + // loong64:"REVB2W" // ppc64x/power10: "BRW" return bits.ReverseBytes32(n) } @@ -219,6 +271,7 @@ func ReverseBytes16(n uint16) uint16 { // arm/5:"SLL","SRL","ORR" // arm/6:"REV16" // arm/7:"REV16" + // loong64:"REVB2H" // ppc64x/power10: "BRH" return bits.ReverseBytes16(n) } @@ -230,7 +283,9 @@ func ReverseBytes16(n uint16) uint16 { func RotateLeft64(n uint64) uint64 { // amd64:"ROLQ" // arm64:"ROR" + // loong64:"ROTRV" // ppc64x:"ROTL" + // riscv64:"RORI" // s390x:"RISBGZ\t[$]0, [$]63, [$]37, " // wasm:"I64Rotl" return bits.RotateLeft64(n, 37) @@ -240,7 +295,9 @@ func RotateLeft32(n uint32) uint32 { // amd64:"ROLL" 386:"ROLL" // arm:`MOVW\tR[0-9]+@>23` // arm64:"RORW" + // loong64:"ROTR\t" // ppc64x:"ROTLW" + // riscv64:"RORIW" // s390x:"RLL" // wasm:"I32Rotl" return bits.RotateLeft32(n, 9) @@ -249,19 +306,23 @@ func RotateLeft32(n uint32) uint32 { func RotateLeft16(n uint16, s int) uint16 { // amd64:"ROLW" 386:"ROLW" // arm64:"RORW",-"CSEL" + // loong64:"ROTR\t","SLLV" return bits.RotateLeft16(n, s) } func RotateLeft8(n uint8, s int) uint8 { // amd64:"ROLB" 386:"ROLB" // arm64:"LSL","LSR",-"CSEL" + // loong64:"OR","SLLV","SRLV" return bits.RotateLeft8(n, s) } func RotateLeftVariable(n uint, m int) uint { // amd64:"ROLQ" // arm64:"ROR" + // loong64:"ROTRV" // ppc64x:"ROTL" + // riscv64:"ROL" // s390x:"RLLG" // wasm:"I64Rotl" return bits.RotateLeft(n, m) @@ -270,7 +331,9 @@ func RotateLeftVariable(n uint, m int) uint { func RotateLeftVariable64(n uint64, m int) uint64 { // amd64:"ROLQ" // arm64:"ROR" + // loong64:"ROTRV" // ppc64x:"ROTL" + // riscv64:"ROL" // s390x:"RLLG" // wasm:"I64Rotl" return bits.RotateLeft64(n, m) @@ -280,7 +343,9 @@ func RotateLeftVariable32(n uint32, m int) uint32 { // arm:`MOVW\tR[0-9]+@>R[0-9]+` // amd64:"ROLL" // arm64:"RORW" + // loong64:"ROTR\t" // ppc64x:"ROTLW" + // riscv64:"ROLW" // s390x:"RLL" // wasm:"I32Rotl" return bits.RotateLeft32(n, m) @@ -296,6 +361,7 @@ func TrailingZeros(n uint) int { // 386:"BSFL" // arm:"CLZ" // arm64:"RBIT","CLZ" + // loong64:"CTZV" // s390x:"FLOGR" // ppc64x/power8:"ANDN","POPCNTD" // ppc64x/power9: "CNTTZD" @@ -308,6 +374,7 @@ func TrailingZeros64(n uint64) int { // amd64/v3:"TZCNTQ" // 386:"BSFL" // arm64:"RBIT","CLZ" + // loong64:"CTZV" // s390x:"FLOGR" // ppc64x/power8:"ANDN","POPCNTD" // ppc64x/power9: "CNTTZD" @@ -327,6 +394,7 @@ func TrailingZeros32(n uint32) int { // 386:"BSFL" // arm:"CLZ" // arm64:"RBITW","CLZW" + // loong64:"CTZW" // s390x:"FLOGR","MOVWZ" // ppc64x/power8:"ANDN","POPCNTW" // ppc64x/power9: "CNTTZW" @@ -339,6 +407,7 @@ func TrailingZeros16(n uint16) int { // 386:"BSFL\t" // arm:"ORR\t\\$65536","CLZ",-"MOVHU\tR" // arm64:"ORR\t\\$65536","RBITW","CLZW",-"MOVHU\tR",-"RBIT\t",-"CLZ\t" + // loong64:"CTZV" // s390x:"FLOGR","OR\t\\$65536" // ppc64x/power8:"POPCNTD","ORIS\\t\\$1" // ppc64x/power9:"CNTTZD","ORIS\\t\\$1" @@ -351,6 +420,7 @@ func TrailingZeros8(n uint8) int { // 386:"BSFL" // arm:"ORR\t\\$256","CLZ",-"MOVBU\tR" // arm64:"ORR\t\\$256","RBITW","CLZW",-"MOVBU\tR",-"RBIT\t",-"CLZ\t" + // loong64:"CTZV" // s390x:"FLOGR","OR\t\\$256" // wasm:"I64Ctz" return bits.TrailingZeros8(n) diff --git a/test/codegen/memcombine.go b/test/codegen/memcombine.go index ff67a442..e1cae0e4 100644 --- a/test/codegen/memcombine.go +++ b/test/codegen/memcombine.go @@ -19,6 +19,7 @@ func load_le64(b []byte) uint64 { // amd64:`MOVQ\s\(.*\),`,-`MOV[BWL]\t[^$]`,-`OR` // s390x:`MOVDBR\s\(.*\),` // arm64:`MOVD\s\(R[0-9]+\),`,-`MOV[BHW]` + // loong64:`MOVBU\s\(R[0-9]+\),` // ppc64le:`MOVD\s`,-`MOV[BHW]Z` // ppc64:`MOVDBR\s`,-`MOV[BHW]Z` return binary.LittleEndian.Uint64(b) @@ -28,6 +29,7 @@ func load_le64_idx(b []byte, idx int) uint64 { // amd64:`MOVQ\s\(.*\)\(.*\*1\),`,-`MOV[BWL]\t[^$]`,-`OR` // s390x:`MOVDBR\s\(.*\)\(.*\*1\),` // arm64:`MOVD\s\(R[0-9]+\)\(R[0-9]+\),`,-`MOV[BHW]` + // loong64:`MOVBU\s\(R[0-9]+\)\(R[0-9]+\),` // ppc64le:`MOVD\s`,-`MOV[BHW]Z\s` // ppc64:`MOVDBR\s`,-`MOV[BHW]Z\s` return binary.LittleEndian.Uint64(b[idx:]) @@ -38,6 +40,7 @@ func load_le32(b []byte) uint32 { // 386:`MOVL\s\(.*\),`,-`MOV[BW]`,-`OR` // s390x:`MOVWBR\s\(.*\),` // arm64:`MOVWU\s\(R[0-9]+\),`,-`MOV[BH]` + // loong64:`MOVBU\s\(R[0-9]+\),` // ppc64le:`MOVWZ\s`,-`MOV[BH]Z\s` // ppc64:`MOVWBR\s`,-`MOV[BH]Z\s` return binary.LittleEndian.Uint32(b) @@ -48,6 +51,7 @@ func load_le32_idx(b []byte, idx int) uint32 { // 386:`MOVL\s\(.*\)\(.*\*1\),`,-`MOV[BW]`,-`OR` // s390x:`MOVWBR\s\(.*\)\(.*\*1\),` // arm64:`MOVWU\s\(R[0-9]+\)\(R[0-9]+\),`,-`MOV[BH]` + // loong64:`MOVBU\s\(R[0-9]+\)\(R[0-9]+\),` // ppc64le:`MOVWZ\s`,-`MOV[BH]Z\s` // ppc64:`MOVWBR\s`,-`MOV[BH]Z\s' return binary.LittleEndian.Uint32(b[idx:]) @@ -57,6 +61,7 @@ func load_le16(b []byte) uint16 { // amd64:`MOVWLZX\s\(.*\),`,-`MOVB`,-`OR` // ppc64le:`MOVHZ\s`,-`MOVBZ` // arm64:`MOVHU\s\(R[0-9]+\),`,-`MOVB` + // loong64:`MOVBU\s\(R[0-9]+\),` // s390x:`MOVHBR\s\(.*\),` // ppc64:`MOVHBR\s`,-`MOVBZ` return binary.LittleEndian.Uint16(b) @@ -67,6 +72,7 @@ func load_le16_idx(b []byte, idx int) uint16 { // ppc64le:`MOVHZ\s`,-`MOVBZ` // ppc64:`MOVHBR\s`,-`MOVBZ` // arm64:`MOVHU\s\(R[0-9]+\)\(R[0-9]+\),`,-`MOVB` + // loong64:`MOVBU\s\(R[0-9]+\)\(R[0-9]+\),` // s390x:`MOVHBR\s\(.*\)\(.*\*1\),` return binary.LittleEndian.Uint16(b[idx:]) } @@ -938,3 +944,29 @@ func issue66413(p *struct { p.c = true p.d = 12 } + +func issue70300(v uint64) (b [8]byte) { + // amd64:"MOVQ",-"MOVB" + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) + b[4] = byte(v >> 32) + b[5] = byte(v >> 40) + b[6] = byte(v >> 48) + b[7] = byte(v >> 56) + return b +} + +func issue70300Reverse(v uint64) (b [8]byte) { + // amd64:"MOVQ",-"MOVB" + b[7] = byte(v >> 56) + b[6] = byte(v >> 48) + b[5] = byte(v >> 40) + b[4] = byte(v >> 32) + b[3] = byte(v >> 24) + b[2] = byte(v >> 16) + b[1] = byte(v >> 8) + b[0] = byte(v) + return b +} diff --git a/test/codegen/shift.go b/test/codegen/shift.go index 6a2a6c40..2d8cf868 100644 --- a/test/codegen/shift.go +++ b/test/codegen/shift.go @@ -462,12 +462,6 @@ func checkMergedShifts64(a [256]uint32, b [256]uint64, c [256]byte, v uint64) { a[2] = a[v>>25&0x7F] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]3, R[0-9]+, [$]29, [$]29, R[0-9]+" a[3] = a[(v>>31)&0x01] - // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" - a[4] = a[(v>>30)&0x07] - // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" - a[5] = a[(v>>32)&0x01] - // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" - a[6] = a[(v>>34)&0x03] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]12, R[0-9]+, [$]21, [$]28, R[0-9]+" b[0] = b[uint8(v>>23)] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]15, R[0-9]+, [$]21, [$]28, R[0-9]+" @@ -476,7 +470,7 @@ func checkMergedShifts64(a [256]uint32, b [256]uint64, c [256]byte, v uint64) { b[2] = b[((uint64((uint32(v) >> 21)) & 0x3f) << 4)] // ppc64x: "RLWNM\t[$]11, R[0-9]+, [$]10, [$]15" c[0] = c[((v>>5)&0x3F)<<16] - // ppc64x: "RLWNM\t[$]0, R[0-9]+, [$]19, [$]24" + // ppc64x: "ANDCC\t[$]8064," c[1] = c[((v>>7)&0x3F)<<7] } @@ -520,3 +514,20 @@ func checkShiftToMask(u []uint64, s []int64) { // amd64:-"SHR",-"SHL","ANDQ" u[1] = u[1] << 5 >> 5 } + +// +// Left shift with addition. +// + +func checkLeftShiftWithAddition(a int64, b int64) int64 { + // riscv64/rva20u64: "SLLI","ADD" + // riscv64/rva22u64: "SH1ADD" + a = a + b<<1 + // riscv64/rva20u64: "SLLI","ADD" + // riscv64/rva22u64: "SH2ADD" + a = a + b<<2 + // riscv64/rva20u64: "SLLI","ADD" + // riscv64/rva22u64: "SH3ADD" + a = a + b<<3 + return a +} diff --git a/test/codegen/slices.go b/test/codegen/slices.go index a38fe77e..9e8990c5 100644 --- a/test/codegen/slices.go +++ b/test/codegen/slices.go @@ -47,6 +47,7 @@ func SliceExtensionConst(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` @@ -58,6 +59,7 @@ func SliceExtensionConstInt64(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` @@ -69,6 +71,7 @@ func SliceExtensionConstUint64(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` @@ -80,16 +83,18 @@ func SliceExtensionConstUint(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` return append(s, make([]int, uint(1<<2))...) } -// On ppc64x continue to use memclrNoHeapPointers +// On ppc64x and loong64 continue to use memclrNoHeapPointers // for sizes >= 512. func SliceExtensionConst512(s []int) []int { // amd64:-`.*runtime\.memclrNoHeapPointers` + // loong64:`.*runtime\.memclrNoHeapPointers` // ppc64x:`.*runtime\.memclrNoHeapPointers` return append(s, make([]int, 1<<9)...) } diff --git a/test/codegen/spills.go b/test/codegen/spills.go new file mode 100644 index 00000000..c8ac9859 --- /dev/null +++ b/test/codegen/spills.go @@ -0,0 +1,31 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func i64(a, b int64) int64 { // arm64:`STP\s`,`LDP\s` + g() + return a + b +} + +func i32(a, b int32) int32 { // arm64:`STPW`,`LDPW` + g() + return a + b +} + +func f64(a, b float64) float64 { // arm64:`FSTPD`,`FLDPD` + g() + return a + b +} + +func f32(a, b float32) float32 { // arm64:`FSTPS`,`FLDPS` + g() + return a + b +} + +//go:noinline +func g() { +} diff --git a/test/codegen/strings.go b/test/codegen/strings.go index f98c062d..64dcf4c8 100644 --- a/test/codegen/strings.go +++ b/test/codegen/strings.go @@ -26,6 +26,11 @@ func ToByteSlice() []byte { // Issue #24698 return []byte("foo") } +func ConvertToByteSlice(a, b, c string) []byte { + // amd64:`.*runtime.concatbyte3` + return []byte(a + b + c) +} + // Loading from read-only symbols should get transformed into constants. func ConstantLoad() { // 12592 = 0x3130 diff --git a/test/codegen/typeswitch.go b/test/codegen/typeswitch.go new file mode 100644 index 00000000..93f8e842 --- /dev/null +++ b/test/codegen/typeswitch.go @@ -0,0 +1,67 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +type Ix interface { + X() +} + +type Iy interface { + Y() +} + +type Iz interface { + Z() +} + +func swXYZ(a Ix) { + switch t := a.(type) { + case Iy: // amd64:-".*typeAssert" + t.Y() + case Iz: // amd64:-".*typeAssert" + t.Z() + } +} + +type Ig[T any] interface { + G() T +} + +func swGYZ[T any](a Ig[T]) { + switch t := a.(type) { + case Iy: // amd64:-".*typeAssert" + t.Y() + case Iz: // amd64:-".*typeAssert" + t.Z() + case interface{ G() T }: // amd64:-".*typeAssert",-".*assertE2I\\(",".*assertE2I2" + t.G() + } +} + +func swE2G[T any](a any) { + switch t := a.(type) { + case Iy: + t.Y() + case Ig[T]: // amd64:-".*assertE2I\\(",".*assertE2I2" + t.G() + } +} + +func swI2G[T any](a Ix) { + switch t := a.(type) { + case Iy: + t.Y() + case Ig[T]: // amd64:-".*assertE2I\\(",".*assertE2I2" + t.G() + } +} + +func swCaller() { + swGYZ[int]((Ig[int])(nil)) + swE2G[int]((Ig[int])(nil)) + swI2G[int]((Ix)(nil)) +} diff --git a/test/codegen/unsafe.go b/test/codegen/unsafe.go new file mode 100644 index 00000000..1f1bdf2a --- /dev/null +++ b/test/codegen/unsafe.go @@ -0,0 +1,16 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +import "unsafe" + +func f(p unsafe.Pointer, x, y uintptr) int64 { + p = unsafe.Pointer(uintptr(p) + x + y) + // amd64:`MOVQ\s\(.*\)\(.*\*1\), ` + // arm64:`MOVD\s\(R[0-9]+\)\(R[0-9]+\), ` + return *(*int64)(p) +} diff --git a/test/codegen/writebarrier.go b/test/codegen/writebarrier.go index 4e0da144..e2b13993 100644 --- a/test/codegen/writebarrier.go +++ b/test/codegen/writebarrier.go @@ -54,6 +54,16 @@ func combine4slice(p *[4][]byte, a, b, c, d []byte) { p[3] = d } +func trickyWriteNil(p *int, q **int) { + if p == nil { + // We change "= p" to "= 0" in the prove pass, which + // means we have one less pointer that needs to go + // into the write barrier buffer. + // amd64:`.*runtime[.]gcWriteBarrier1` + *q = p + } +} + type S struct { a, b string c *int diff --git a/test/defererrcheck.go b/test/defererrcheck.go index 95b91da5..c20f090b 100644 --- a/test/defererrcheck.go +++ b/test/defererrcheck.go @@ -84,3 +84,57 @@ label: fmt.Println("defer") }() } + +// Test for function with too many exits, which will disable open-coded defer +// even though the number of defer statements is not greater than 8. +func f7() { + defer println(1) // ERROR "open-coded defer" + defer println(1) // ERROR "open-coded defer" + defer println(1) // ERROR "open-coded defer" + defer println(1) // ERROR "open-coded defer" + + switch glob { + case 1: + return + case 2: + return + case 3: + return + } +} + +func f8() { + defer println(1) // ERROR "stack-allocated defer" + defer println(1) // ERROR "stack-allocated defer" + defer println(1) // ERROR "stack-allocated defer" + defer println(1) // ERROR "stack-allocated defer" + + switch glob { + case 1: + return + case 2: + return + case 3: + return + case 4: + return + } +} + +func f9() { + defer println(1) // ERROR "open-coded defer" + defer println(1) // ERROR "open-coded defer" + defer println(1) // ERROR "open-coded defer" + defer println(1) // ERROR "open-coded defer" + + switch glob { + case 1: + return + case 2: + return + case 3: + return + case 4: + panic("") + } +} diff --git a/test/escape4.go b/test/escape4.go index c4a2fc15..ddeaff81 100644 --- a/test/escape4.go +++ b/test/escape4.go @@ -25,7 +25,7 @@ func f1() { // Escape analysis used to miss inlined code in closures. func() { // ERROR "can inline f1.func1" - p = alloc(3) // ERROR "inlining call to alloc" + p = alloc(3) // ERROR "inlining call to alloc" "moved to heap: x" }() // ERROR "inlining call to f1.func1" "inlining call to alloc" "moved to heap: x" f = func() { // ERROR "func literal escapes to heap" "can inline f1.func2" diff --git a/test/escape_closure.go b/test/escape_closure.go index 0b19d6f6..84f3adf3 100644 --- a/test/escape_closure.go +++ b/test/escape_closure.go @@ -134,7 +134,7 @@ func ClosureCallArgs14() { func ClosureCallArgs15() { x := 0 // ERROR "moved to heap: x" p := &x - sink = func(p **int) *int { // ERROR "leaking param content: p" "func literal does not escape" + sink = func(p **int) *int { // ERROR "leaking param: p to result ~r0 level=1" "func literal does not escape" return *p }(&p) } diff --git a/test/escape_reflect.go b/test/escape_reflect.go index 99fbada9..e5032370 100644 --- a/test/escape_reflect.go +++ b/test/escape_reflect.go @@ -423,7 +423,7 @@ func mapdelete(m map[string]string, k string) { // ERROR "m does not escape" "le } // Unfortunate: v doesn't need to leak. -func setiterkey1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "it does not escape" +func setiterkey1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "leaking param content: it$" v.SetIterKey(it) } @@ -434,7 +434,7 @@ func setiterkey2(v reflect.Value, m map[string]string) { // ERROR "leaking param } // Unfortunate: v doesn't need to leak. -func setitervalue1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "it does not escape" +func setitervalue1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "leaking param content: it$" v.SetIterValue(it) } diff --git a/test/fixedbugs/bug195.go b/test/fixedbugs/bug195.go index 769ed050..fa9cec75 100644 --- a/test/fixedbugs/bug195.go +++ b/test/fixedbugs/bug195.go @@ -18,7 +18,7 @@ type I4 interface { // GC_ERROR "invalid recursive type: I4 refers to itself" I4 // GCCGO_ERROR "interface" } -type I5 interface { // GC_ERROR "invalid recursive type I5\n\tLINE:.* I5 refers to\n\tLINE+4:.* I6 refers to\n\tLINE:.* I5$" +type I5 interface { // GC_ERROR "invalid recursive type I5\n\tLINE:.* I5 refers to I6\n\tLINE+4:.* I6 refers to I5$" I6 } diff --git a/test/fixedbugs/bug385_64.go b/test/fixedbugs/bug385_64.go index deba9c9f..49201d95 100644 --- a/test/fixedbugs/bug385_64.go +++ b/test/fixedbugs/bug385_64.go @@ -11,423 +11,49449 @@ package main -var z [10 << 20]byte +// seq 1 16480 | sed 's/.*/var z& [1 << 17]byte/' +var z1 [1 << 17]byte +var z2 [1 << 17]byte +var z3 [1 << 17]byte +var z4 [1 << 17]byte +var z5 [1 << 17]byte +var z6 [1 << 17]byte +var z7 [1 << 17]byte +var z8 [1 << 17]byte +var z9 [1 << 17]byte +var z10 [1 << 17]byte +var z11 [1 << 17]byte +var z12 [1 << 17]byte +var z13 [1 << 17]byte +var z14 [1 << 17]byte +var z15 [1 << 17]byte +var z16 [1 << 17]byte +var z17 [1 << 17]byte +var z18 [1 << 17]byte +var z19 [1 << 17]byte +var z20 [1 << 17]byte +var z21 [1 << 17]byte +var z22 [1 << 17]byte +var z23 [1 << 17]byte +var z24 [1 << 17]byte +var z25 [1 << 17]byte +var z26 [1 << 17]byte +var z27 [1 << 17]byte +var z28 [1 << 17]byte +var z29 [1 << 17]byte +var z30 [1 << 17]byte +var z31 [1 << 17]byte +var z32 [1 << 17]byte +var z33 [1 << 17]byte +var z34 [1 << 17]byte +var z35 [1 << 17]byte +var z36 [1 << 17]byte +var z37 [1 << 17]byte +var z38 [1 << 17]byte +var z39 [1 << 17]byte +var z40 [1 << 17]byte +var z41 [1 << 17]byte +var z42 [1 << 17]byte +var z43 [1 << 17]byte +var z44 [1 << 17]byte +var z45 [1 << 17]byte +var z46 [1 << 17]byte +var z47 [1 << 17]byte +var z48 [1 << 17]byte +var z49 [1 << 17]byte +var z50 [1 << 17]byte +var z51 [1 << 17]byte +var z52 [1 << 17]byte +var z53 [1 << 17]byte +var z54 [1 << 17]byte +var z55 [1 << 17]byte +var z56 [1 << 17]byte +var z57 [1 << 17]byte +var z58 [1 << 17]byte +var z59 [1 << 17]byte +var z60 [1 << 17]byte +var z61 [1 << 17]byte +var z62 [1 << 17]byte +var z63 [1 << 17]byte +var z64 [1 << 17]byte +var z65 [1 << 17]byte +var z66 [1 << 17]byte +var z67 [1 << 17]byte +var z68 [1 << 17]byte +var z69 [1 << 17]byte +var z70 [1 << 17]byte +var z71 [1 << 17]byte +var z72 [1 << 17]byte +var z73 [1 << 17]byte +var z74 [1 << 17]byte +var z75 [1 << 17]byte +var z76 [1 << 17]byte +var z77 [1 << 17]byte +var z78 [1 << 17]byte +var z79 [1 << 17]byte +var z80 [1 << 17]byte +var z81 [1 << 17]byte +var z82 [1 << 17]byte +var z83 [1 << 17]byte +var z84 [1 << 17]byte +var z85 [1 << 17]byte +var z86 [1 << 17]byte +var z87 [1 << 17]byte +var z88 [1 << 17]byte +var z89 [1 << 17]byte +var z90 [1 << 17]byte +var z91 [1 << 17]byte +var z92 [1 << 17]byte +var z93 [1 << 17]byte +var z94 [1 << 17]byte +var z95 [1 << 17]byte +var z96 [1 << 17]byte +var z97 [1 << 17]byte +var z98 [1 << 17]byte +var z99 [1 << 17]byte +var z100 [1 << 17]byte +var z101 [1 << 17]byte +var z102 [1 << 17]byte +var z103 [1 << 17]byte +var z104 [1 << 17]byte +var z105 [1 << 17]byte +var z106 [1 << 17]byte +var z107 [1 << 17]byte +var z108 [1 << 17]byte +var z109 [1 << 17]byte +var z110 [1 << 17]byte +var z111 [1 << 17]byte +var z112 [1 << 17]byte +var z113 [1 << 17]byte +var z114 [1 << 17]byte +var z115 [1 << 17]byte +var z116 [1 << 17]byte +var z117 [1 << 17]byte +var z118 [1 << 17]byte +var z119 [1 << 17]byte +var z120 [1 << 17]byte +var z121 [1 << 17]byte +var z122 [1 << 17]byte +var z123 [1 << 17]byte +var z124 [1 << 17]byte +var z125 [1 << 17]byte +var z126 [1 << 17]byte +var z127 [1 << 17]byte +var z128 [1 << 17]byte +var z129 [1 << 17]byte +var z130 [1 << 17]byte +var z131 [1 << 17]byte +var z132 [1 << 17]byte +var z133 [1 << 17]byte +var z134 [1 << 17]byte +var z135 [1 << 17]byte +var z136 [1 << 17]byte +var z137 [1 << 17]byte +var z138 [1 << 17]byte +var z139 [1 << 17]byte +var z140 [1 << 17]byte +var z141 [1 << 17]byte +var z142 [1 << 17]byte +var z143 [1 << 17]byte +var z144 [1 << 17]byte +var z145 [1 << 17]byte +var z146 [1 << 17]byte +var z147 [1 << 17]byte +var z148 [1 << 17]byte +var z149 [1 << 17]byte +var z150 [1 << 17]byte +var z151 [1 << 17]byte +var z152 [1 << 17]byte +var z153 [1 << 17]byte +var z154 [1 << 17]byte +var z155 [1 << 17]byte +var z156 [1 << 17]byte +var z157 [1 << 17]byte +var z158 [1 << 17]byte +var z159 [1 << 17]byte +var z160 [1 << 17]byte +var z161 [1 << 17]byte +var z162 [1 << 17]byte +var z163 [1 << 17]byte +var z164 [1 << 17]byte +var z165 [1 << 17]byte +var z166 [1 << 17]byte +var z167 [1 << 17]byte +var z168 [1 << 17]byte +var z169 [1 << 17]byte +var z170 [1 << 17]byte +var z171 [1 << 17]byte +var z172 [1 << 17]byte +var z173 [1 << 17]byte +var z174 [1 << 17]byte +var z175 [1 << 17]byte +var z176 [1 << 17]byte +var z177 [1 << 17]byte +var z178 [1 << 17]byte +var z179 [1 << 17]byte +var z180 [1 << 17]byte +var z181 [1 << 17]byte +var z182 [1 << 17]byte +var z183 [1 << 17]byte +var z184 [1 << 17]byte +var z185 [1 << 17]byte +var z186 [1 << 17]byte +var z187 [1 << 17]byte +var z188 [1 << 17]byte +var z189 [1 << 17]byte +var z190 [1 << 17]byte +var z191 [1 << 17]byte +var z192 [1 << 17]byte +var z193 [1 << 17]byte +var z194 [1 << 17]byte +var z195 [1 << 17]byte +var z196 [1 << 17]byte +var z197 [1 << 17]byte +var z198 [1 << 17]byte +var z199 [1 << 17]byte +var z200 [1 << 17]byte +var z201 [1 << 17]byte +var z202 [1 << 17]byte +var z203 [1 << 17]byte +var z204 [1 << 17]byte +var z205 [1 << 17]byte +var z206 [1 << 17]byte +var z207 [1 << 17]byte +var z208 [1 << 17]byte +var z209 [1 << 17]byte +var z210 [1 << 17]byte +var z211 [1 << 17]byte +var z212 [1 << 17]byte +var z213 [1 << 17]byte +var z214 [1 << 17]byte +var z215 [1 << 17]byte +var z216 [1 << 17]byte +var z217 [1 << 17]byte +var z218 [1 << 17]byte +var z219 [1 << 17]byte +var z220 [1 << 17]byte +var z221 [1 << 17]byte +var z222 [1 << 17]byte +var z223 [1 << 17]byte +var z224 [1 << 17]byte +var z225 [1 << 17]byte +var z226 [1 << 17]byte +var z227 [1 << 17]byte +var z228 [1 << 17]byte +var z229 [1 << 17]byte +var z230 [1 << 17]byte +var z231 [1 << 17]byte +var z232 [1 << 17]byte +var z233 [1 << 17]byte +var z234 [1 << 17]byte +var z235 [1 << 17]byte +var z236 [1 << 17]byte +var z237 [1 << 17]byte +var z238 [1 << 17]byte +var z239 [1 << 17]byte +var z240 [1 << 17]byte +var z241 [1 << 17]byte +var z242 [1 << 17]byte +var z243 [1 << 17]byte +var z244 [1 << 17]byte +var z245 [1 << 17]byte +var z246 [1 << 17]byte +var z247 [1 << 17]byte +var z248 [1 << 17]byte +var z249 [1 << 17]byte +var z250 [1 << 17]byte +var z251 [1 << 17]byte +var z252 [1 << 17]byte +var z253 [1 << 17]byte +var z254 [1 << 17]byte +var z255 [1 << 17]byte +var z256 [1 << 17]byte +var z257 [1 << 17]byte +var z258 [1 << 17]byte +var z259 [1 << 17]byte +var z260 [1 << 17]byte +var z261 [1 << 17]byte +var z262 [1 << 17]byte +var z263 [1 << 17]byte +var z264 [1 << 17]byte +var z265 [1 << 17]byte +var z266 [1 << 17]byte +var z267 [1 << 17]byte +var z268 [1 << 17]byte +var z269 [1 << 17]byte +var z270 [1 << 17]byte +var z271 [1 << 17]byte +var z272 [1 << 17]byte +var z273 [1 << 17]byte +var z274 [1 << 17]byte +var z275 [1 << 17]byte +var z276 [1 << 17]byte +var z277 [1 << 17]byte +var z278 [1 << 17]byte +var z279 [1 << 17]byte +var z280 [1 << 17]byte +var z281 [1 << 17]byte +var z282 [1 << 17]byte +var z283 [1 << 17]byte +var z284 [1 << 17]byte +var z285 [1 << 17]byte +var z286 [1 << 17]byte +var z287 [1 << 17]byte +var z288 [1 << 17]byte +var z289 [1 << 17]byte +var z290 [1 << 17]byte +var z291 [1 << 17]byte +var z292 [1 << 17]byte +var z293 [1 << 17]byte +var z294 [1 << 17]byte +var z295 [1 << 17]byte +var z296 [1 << 17]byte +var z297 [1 << 17]byte +var z298 [1 << 17]byte +var z299 [1 << 17]byte +var z300 [1 << 17]byte +var z301 [1 << 17]byte +var z302 [1 << 17]byte +var z303 [1 << 17]byte +var z304 [1 << 17]byte +var z305 [1 << 17]byte +var z306 [1 << 17]byte +var z307 [1 << 17]byte +var z308 [1 << 17]byte +var z309 [1 << 17]byte +var z310 [1 << 17]byte +var z311 [1 << 17]byte +var z312 [1 << 17]byte +var z313 [1 << 17]byte +var z314 [1 << 17]byte +var z315 [1 << 17]byte +var z316 [1 << 17]byte +var z317 [1 << 17]byte +var z318 [1 << 17]byte +var z319 [1 << 17]byte +var z320 [1 << 17]byte +var z321 [1 << 17]byte +var z322 [1 << 17]byte +var z323 [1 << 17]byte +var z324 [1 << 17]byte +var z325 [1 << 17]byte +var z326 [1 << 17]byte +var z327 [1 << 17]byte +var z328 [1 << 17]byte +var z329 [1 << 17]byte +var z330 [1 << 17]byte +var z331 [1 << 17]byte +var z332 [1 << 17]byte +var z333 [1 << 17]byte +var z334 [1 << 17]byte +var z335 [1 << 17]byte +var z336 [1 << 17]byte +var z337 [1 << 17]byte +var z338 [1 << 17]byte +var z339 [1 << 17]byte +var z340 [1 << 17]byte +var z341 [1 << 17]byte +var z342 [1 << 17]byte +var z343 [1 << 17]byte +var z344 [1 << 17]byte +var z345 [1 << 17]byte +var z346 [1 << 17]byte +var z347 [1 << 17]byte +var z348 [1 << 17]byte +var z349 [1 << 17]byte +var z350 [1 << 17]byte +var z351 [1 << 17]byte +var z352 [1 << 17]byte +var z353 [1 << 17]byte +var z354 [1 << 17]byte +var z355 [1 << 17]byte +var z356 [1 << 17]byte +var z357 [1 << 17]byte +var z358 [1 << 17]byte +var z359 [1 << 17]byte +var z360 [1 << 17]byte +var z361 [1 << 17]byte +var z362 [1 << 17]byte +var z363 [1 << 17]byte +var z364 [1 << 17]byte +var z365 [1 << 17]byte +var z366 [1 << 17]byte +var z367 [1 << 17]byte +var z368 [1 << 17]byte +var z369 [1 << 17]byte +var z370 [1 << 17]byte +var z371 [1 << 17]byte +var z372 [1 << 17]byte +var z373 [1 << 17]byte +var z374 [1 << 17]byte +var z375 [1 << 17]byte +var z376 [1 << 17]byte +var z377 [1 << 17]byte +var z378 [1 << 17]byte +var z379 [1 << 17]byte +var z380 [1 << 17]byte +var z381 [1 << 17]byte +var z382 [1 << 17]byte +var z383 [1 << 17]byte +var z384 [1 << 17]byte +var z385 [1 << 17]byte +var z386 [1 << 17]byte +var z387 [1 << 17]byte +var z388 [1 << 17]byte +var z389 [1 << 17]byte +var z390 [1 << 17]byte +var z391 [1 << 17]byte +var z392 [1 << 17]byte +var z393 [1 << 17]byte +var z394 [1 << 17]byte +var z395 [1 << 17]byte +var z396 [1 << 17]byte +var z397 [1 << 17]byte +var z398 [1 << 17]byte +var z399 [1 << 17]byte +var z400 [1 << 17]byte +var z401 [1 << 17]byte +var z402 [1 << 17]byte +var z403 [1 << 17]byte +var z404 [1 << 17]byte +var z405 [1 << 17]byte +var z406 [1 << 17]byte +var z407 [1 << 17]byte +var z408 [1 << 17]byte +var z409 [1 << 17]byte +var z410 [1 << 17]byte +var z411 [1 << 17]byte +var z412 [1 << 17]byte +var z413 [1 << 17]byte +var z414 [1 << 17]byte +var z415 [1 << 17]byte +var z416 [1 << 17]byte +var z417 [1 << 17]byte +var z418 [1 << 17]byte +var z419 [1 << 17]byte +var z420 [1 << 17]byte +var z421 [1 << 17]byte +var z422 [1 << 17]byte +var z423 [1 << 17]byte +var z424 [1 << 17]byte +var z425 [1 << 17]byte +var z426 [1 << 17]byte +var z427 [1 << 17]byte +var z428 [1 << 17]byte +var z429 [1 << 17]byte +var z430 [1 << 17]byte +var z431 [1 << 17]byte +var z432 [1 << 17]byte +var z433 [1 << 17]byte +var z434 [1 << 17]byte +var z435 [1 << 17]byte +var z436 [1 << 17]byte +var z437 [1 << 17]byte +var z438 [1 << 17]byte +var z439 [1 << 17]byte +var z440 [1 << 17]byte +var z441 [1 << 17]byte +var z442 [1 << 17]byte +var z443 [1 << 17]byte +var z444 [1 << 17]byte +var z445 [1 << 17]byte +var z446 [1 << 17]byte +var z447 [1 << 17]byte +var z448 [1 << 17]byte +var z449 [1 << 17]byte +var z450 [1 << 17]byte +var z451 [1 << 17]byte +var z452 [1 << 17]byte +var z453 [1 << 17]byte +var z454 [1 << 17]byte +var z455 [1 << 17]byte +var z456 [1 << 17]byte +var z457 [1 << 17]byte +var z458 [1 << 17]byte +var z459 [1 << 17]byte +var z460 [1 << 17]byte +var z461 [1 << 17]byte +var z462 [1 << 17]byte +var z463 [1 << 17]byte +var z464 [1 << 17]byte +var z465 [1 << 17]byte +var z466 [1 << 17]byte +var z467 [1 << 17]byte +var z468 [1 << 17]byte +var z469 [1 << 17]byte +var z470 [1 << 17]byte +var z471 [1 << 17]byte +var z472 [1 << 17]byte +var z473 [1 << 17]byte +var z474 [1 << 17]byte +var z475 [1 << 17]byte +var z476 [1 << 17]byte +var z477 [1 << 17]byte +var z478 [1 << 17]byte +var z479 [1 << 17]byte +var z480 [1 << 17]byte +var z481 [1 << 17]byte +var z482 [1 << 17]byte +var z483 [1 << 17]byte +var z484 [1 << 17]byte +var z485 [1 << 17]byte +var z486 [1 << 17]byte +var z487 [1 << 17]byte +var z488 [1 << 17]byte +var z489 [1 << 17]byte +var z490 [1 << 17]byte +var z491 [1 << 17]byte +var z492 [1 << 17]byte +var z493 [1 << 17]byte +var z494 [1 << 17]byte +var z495 [1 << 17]byte +var z496 [1 << 17]byte +var z497 [1 << 17]byte +var z498 [1 << 17]byte +var z499 [1 << 17]byte +var z500 [1 << 17]byte +var z501 [1 << 17]byte +var z502 [1 << 17]byte +var z503 [1 << 17]byte +var z504 [1 << 17]byte +var z505 [1 << 17]byte +var z506 [1 << 17]byte +var z507 [1 << 17]byte +var z508 [1 << 17]byte +var z509 [1 << 17]byte +var z510 [1 << 17]byte +var z511 [1 << 17]byte +var z512 [1 << 17]byte +var z513 [1 << 17]byte +var z514 [1 << 17]byte +var z515 [1 << 17]byte +var z516 [1 << 17]byte +var z517 [1 << 17]byte +var z518 [1 << 17]byte +var z519 [1 << 17]byte +var z520 [1 << 17]byte +var z521 [1 << 17]byte +var z522 [1 << 17]byte +var z523 [1 << 17]byte +var z524 [1 << 17]byte +var z525 [1 << 17]byte +var z526 [1 << 17]byte +var z527 [1 << 17]byte +var z528 [1 << 17]byte +var z529 [1 << 17]byte +var z530 [1 << 17]byte +var z531 [1 << 17]byte +var z532 [1 << 17]byte +var z533 [1 << 17]byte +var z534 [1 << 17]byte +var z535 [1 << 17]byte +var z536 [1 << 17]byte +var z537 [1 << 17]byte +var z538 [1 << 17]byte +var z539 [1 << 17]byte +var z540 [1 << 17]byte +var z541 [1 << 17]byte +var z542 [1 << 17]byte +var z543 [1 << 17]byte +var z544 [1 << 17]byte +var z545 [1 << 17]byte +var z546 [1 << 17]byte +var z547 [1 << 17]byte +var z548 [1 << 17]byte +var z549 [1 << 17]byte +var z550 [1 << 17]byte +var z551 [1 << 17]byte +var z552 [1 << 17]byte +var z553 [1 << 17]byte +var z554 [1 << 17]byte +var z555 [1 << 17]byte +var z556 [1 << 17]byte +var z557 [1 << 17]byte +var z558 [1 << 17]byte +var z559 [1 << 17]byte +var z560 [1 << 17]byte +var z561 [1 << 17]byte +var z562 [1 << 17]byte +var z563 [1 << 17]byte +var z564 [1 << 17]byte +var z565 [1 << 17]byte +var z566 [1 << 17]byte +var z567 [1 << 17]byte +var z568 [1 << 17]byte +var z569 [1 << 17]byte +var z570 [1 << 17]byte +var z571 [1 << 17]byte +var z572 [1 << 17]byte +var z573 [1 << 17]byte +var z574 [1 << 17]byte +var z575 [1 << 17]byte +var z576 [1 << 17]byte +var z577 [1 << 17]byte +var z578 [1 << 17]byte +var z579 [1 << 17]byte +var z580 [1 << 17]byte +var z581 [1 << 17]byte +var z582 [1 << 17]byte +var z583 [1 << 17]byte +var z584 [1 << 17]byte +var z585 [1 << 17]byte +var z586 [1 << 17]byte +var z587 [1 << 17]byte +var z588 [1 << 17]byte +var z589 [1 << 17]byte +var z590 [1 << 17]byte +var z591 [1 << 17]byte +var z592 [1 << 17]byte +var z593 [1 << 17]byte +var z594 [1 << 17]byte +var z595 [1 << 17]byte +var z596 [1 << 17]byte +var z597 [1 << 17]byte +var z598 [1 << 17]byte +var z599 [1 << 17]byte +var z600 [1 << 17]byte +var z601 [1 << 17]byte +var z602 [1 << 17]byte +var z603 [1 << 17]byte +var z604 [1 << 17]byte +var z605 [1 << 17]byte +var z606 [1 << 17]byte +var z607 [1 << 17]byte +var z608 [1 << 17]byte +var z609 [1 << 17]byte +var z610 [1 << 17]byte +var z611 [1 << 17]byte +var z612 [1 << 17]byte +var z613 [1 << 17]byte +var z614 [1 << 17]byte +var z615 [1 << 17]byte +var z616 [1 << 17]byte +var z617 [1 << 17]byte +var z618 [1 << 17]byte +var z619 [1 << 17]byte +var z620 [1 << 17]byte +var z621 [1 << 17]byte +var z622 [1 << 17]byte +var z623 [1 << 17]byte +var z624 [1 << 17]byte +var z625 [1 << 17]byte +var z626 [1 << 17]byte +var z627 [1 << 17]byte +var z628 [1 << 17]byte +var z629 [1 << 17]byte +var z630 [1 << 17]byte +var z631 [1 << 17]byte +var z632 [1 << 17]byte +var z633 [1 << 17]byte +var z634 [1 << 17]byte +var z635 [1 << 17]byte +var z636 [1 << 17]byte +var z637 [1 << 17]byte +var z638 [1 << 17]byte +var z639 [1 << 17]byte +var z640 [1 << 17]byte +var z641 [1 << 17]byte +var z642 [1 << 17]byte +var z643 [1 << 17]byte +var z644 [1 << 17]byte +var z645 [1 << 17]byte +var z646 [1 << 17]byte +var z647 [1 << 17]byte +var z648 [1 << 17]byte +var z649 [1 << 17]byte +var z650 [1 << 17]byte +var z651 [1 << 17]byte +var z652 [1 << 17]byte +var z653 [1 << 17]byte +var z654 [1 << 17]byte +var z655 [1 << 17]byte +var z656 [1 << 17]byte +var z657 [1 << 17]byte +var z658 [1 << 17]byte +var z659 [1 << 17]byte +var z660 [1 << 17]byte +var z661 [1 << 17]byte +var z662 [1 << 17]byte +var z663 [1 << 17]byte +var z664 [1 << 17]byte +var z665 [1 << 17]byte +var z666 [1 << 17]byte +var z667 [1 << 17]byte +var z668 [1 << 17]byte +var z669 [1 << 17]byte +var z670 [1 << 17]byte +var z671 [1 << 17]byte +var z672 [1 << 17]byte +var z673 [1 << 17]byte +var z674 [1 << 17]byte +var z675 [1 << 17]byte +var z676 [1 << 17]byte +var z677 [1 << 17]byte +var z678 [1 << 17]byte +var z679 [1 << 17]byte +var z680 [1 << 17]byte +var z681 [1 << 17]byte +var z682 [1 << 17]byte +var z683 [1 << 17]byte +var z684 [1 << 17]byte +var z685 [1 << 17]byte +var z686 [1 << 17]byte +var z687 [1 << 17]byte +var z688 [1 << 17]byte +var z689 [1 << 17]byte +var z690 [1 << 17]byte +var z691 [1 << 17]byte +var z692 [1 << 17]byte +var z693 [1 << 17]byte +var z694 [1 << 17]byte +var z695 [1 << 17]byte +var z696 [1 << 17]byte +var z697 [1 << 17]byte +var z698 [1 << 17]byte +var z699 [1 << 17]byte +var z700 [1 << 17]byte +var z701 [1 << 17]byte +var z702 [1 << 17]byte +var z703 [1 << 17]byte +var z704 [1 << 17]byte +var z705 [1 << 17]byte +var z706 [1 << 17]byte +var z707 [1 << 17]byte +var z708 [1 << 17]byte +var z709 [1 << 17]byte +var z710 [1 << 17]byte +var z711 [1 << 17]byte +var z712 [1 << 17]byte +var z713 [1 << 17]byte +var z714 [1 << 17]byte +var z715 [1 << 17]byte +var z716 [1 << 17]byte +var z717 [1 << 17]byte +var z718 [1 << 17]byte +var z719 [1 << 17]byte +var z720 [1 << 17]byte +var z721 [1 << 17]byte +var z722 [1 << 17]byte +var z723 [1 << 17]byte +var z724 [1 << 17]byte +var z725 [1 << 17]byte +var z726 [1 << 17]byte +var z727 [1 << 17]byte +var z728 [1 << 17]byte +var z729 [1 << 17]byte +var z730 [1 << 17]byte +var z731 [1 << 17]byte +var z732 [1 << 17]byte +var z733 [1 << 17]byte +var z734 [1 << 17]byte +var z735 [1 << 17]byte +var z736 [1 << 17]byte +var z737 [1 << 17]byte +var z738 [1 << 17]byte +var z739 [1 << 17]byte +var z740 [1 << 17]byte +var z741 [1 << 17]byte +var z742 [1 << 17]byte +var z743 [1 << 17]byte +var z744 [1 << 17]byte +var z745 [1 << 17]byte +var z746 [1 << 17]byte +var z747 [1 << 17]byte +var z748 [1 << 17]byte +var z749 [1 << 17]byte +var z750 [1 << 17]byte +var z751 [1 << 17]byte +var z752 [1 << 17]byte +var z753 [1 << 17]byte +var z754 [1 << 17]byte +var z755 [1 << 17]byte +var z756 [1 << 17]byte +var z757 [1 << 17]byte +var z758 [1 << 17]byte +var z759 [1 << 17]byte +var z760 [1 << 17]byte +var z761 [1 << 17]byte +var z762 [1 << 17]byte +var z763 [1 << 17]byte +var z764 [1 << 17]byte +var z765 [1 << 17]byte +var z766 [1 << 17]byte +var z767 [1 << 17]byte +var z768 [1 << 17]byte +var z769 [1 << 17]byte +var z770 [1 << 17]byte +var z771 [1 << 17]byte +var z772 [1 << 17]byte +var z773 [1 << 17]byte +var z774 [1 << 17]byte +var z775 [1 << 17]byte +var z776 [1 << 17]byte +var z777 [1 << 17]byte +var z778 [1 << 17]byte +var z779 [1 << 17]byte +var z780 [1 << 17]byte +var z781 [1 << 17]byte +var z782 [1 << 17]byte +var z783 [1 << 17]byte +var z784 [1 << 17]byte +var z785 [1 << 17]byte +var z786 [1 << 17]byte +var z787 [1 << 17]byte +var z788 [1 << 17]byte +var z789 [1 << 17]byte +var z790 [1 << 17]byte +var z791 [1 << 17]byte +var z792 [1 << 17]byte +var z793 [1 << 17]byte +var z794 [1 << 17]byte +var z795 [1 << 17]byte +var z796 [1 << 17]byte +var z797 [1 << 17]byte +var z798 [1 << 17]byte +var z799 [1 << 17]byte +var z800 [1 << 17]byte +var z801 [1 << 17]byte +var z802 [1 << 17]byte +var z803 [1 << 17]byte +var z804 [1 << 17]byte +var z805 [1 << 17]byte +var z806 [1 << 17]byte +var z807 [1 << 17]byte +var z808 [1 << 17]byte +var z809 [1 << 17]byte +var z810 [1 << 17]byte +var z811 [1 << 17]byte +var z812 [1 << 17]byte +var z813 [1 << 17]byte +var z814 [1 << 17]byte +var z815 [1 << 17]byte +var z816 [1 << 17]byte +var z817 [1 << 17]byte +var z818 [1 << 17]byte +var z819 [1 << 17]byte +var z820 [1 << 17]byte +var z821 [1 << 17]byte +var z822 [1 << 17]byte +var z823 [1 << 17]byte +var z824 [1 << 17]byte +var z825 [1 << 17]byte +var z826 [1 << 17]byte +var z827 [1 << 17]byte +var z828 [1 << 17]byte +var z829 [1 << 17]byte +var z830 [1 << 17]byte +var z831 [1 << 17]byte +var z832 [1 << 17]byte +var z833 [1 << 17]byte +var z834 [1 << 17]byte +var z835 [1 << 17]byte +var z836 [1 << 17]byte +var z837 [1 << 17]byte +var z838 [1 << 17]byte +var z839 [1 << 17]byte +var z840 [1 << 17]byte +var z841 [1 << 17]byte +var z842 [1 << 17]byte +var z843 [1 << 17]byte +var z844 [1 << 17]byte +var z845 [1 << 17]byte +var z846 [1 << 17]byte +var z847 [1 << 17]byte +var z848 [1 << 17]byte +var z849 [1 << 17]byte +var z850 [1 << 17]byte +var z851 [1 << 17]byte +var z852 [1 << 17]byte +var z853 [1 << 17]byte +var z854 [1 << 17]byte +var z855 [1 << 17]byte +var z856 [1 << 17]byte +var z857 [1 << 17]byte +var z858 [1 << 17]byte +var z859 [1 << 17]byte +var z860 [1 << 17]byte +var z861 [1 << 17]byte +var z862 [1 << 17]byte +var z863 [1 << 17]byte +var z864 [1 << 17]byte +var z865 [1 << 17]byte +var z866 [1 << 17]byte +var z867 [1 << 17]byte +var z868 [1 << 17]byte +var z869 [1 << 17]byte +var z870 [1 << 17]byte +var z871 [1 << 17]byte +var z872 [1 << 17]byte +var z873 [1 << 17]byte +var z874 [1 << 17]byte +var z875 [1 << 17]byte +var z876 [1 << 17]byte +var z877 [1 << 17]byte +var z878 [1 << 17]byte +var z879 [1 << 17]byte +var z880 [1 << 17]byte +var z881 [1 << 17]byte +var z882 [1 << 17]byte +var z883 [1 << 17]byte +var z884 [1 << 17]byte +var z885 [1 << 17]byte +var z886 [1 << 17]byte +var z887 [1 << 17]byte +var z888 [1 << 17]byte +var z889 [1 << 17]byte +var z890 [1 << 17]byte +var z891 [1 << 17]byte +var z892 [1 << 17]byte +var z893 [1 << 17]byte +var z894 [1 << 17]byte +var z895 [1 << 17]byte +var z896 [1 << 17]byte +var z897 [1 << 17]byte +var z898 [1 << 17]byte +var z899 [1 << 17]byte +var z900 [1 << 17]byte +var z901 [1 << 17]byte +var z902 [1 << 17]byte +var z903 [1 << 17]byte +var z904 [1 << 17]byte +var z905 [1 << 17]byte +var z906 [1 << 17]byte +var z907 [1 << 17]byte +var z908 [1 << 17]byte +var z909 [1 << 17]byte +var z910 [1 << 17]byte +var z911 [1 << 17]byte +var z912 [1 << 17]byte +var z913 [1 << 17]byte +var z914 [1 << 17]byte +var z915 [1 << 17]byte +var z916 [1 << 17]byte +var z917 [1 << 17]byte +var z918 [1 << 17]byte +var z919 [1 << 17]byte +var z920 [1 << 17]byte +var z921 [1 << 17]byte +var z922 [1 << 17]byte +var z923 [1 << 17]byte +var z924 [1 << 17]byte +var z925 [1 << 17]byte +var z926 [1 << 17]byte +var z927 [1 << 17]byte +var z928 [1 << 17]byte +var z929 [1 << 17]byte +var z930 [1 << 17]byte +var z931 [1 << 17]byte +var z932 [1 << 17]byte +var z933 [1 << 17]byte +var z934 [1 << 17]byte +var z935 [1 << 17]byte +var z936 [1 << 17]byte +var z937 [1 << 17]byte +var z938 [1 << 17]byte +var z939 [1 << 17]byte +var z940 [1 << 17]byte +var z941 [1 << 17]byte +var z942 [1 << 17]byte +var z943 [1 << 17]byte +var z944 [1 << 17]byte +var z945 [1 << 17]byte +var z946 [1 << 17]byte +var z947 [1 << 17]byte +var z948 [1 << 17]byte +var z949 [1 << 17]byte +var z950 [1 << 17]byte +var z951 [1 << 17]byte +var z952 [1 << 17]byte +var z953 [1 << 17]byte +var z954 [1 << 17]byte +var z955 [1 << 17]byte +var z956 [1 << 17]byte +var z957 [1 << 17]byte +var z958 [1 << 17]byte +var z959 [1 << 17]byte +var z960 [1 << 17]byte +var z961 [1 << 17]byte +var z962 [1 << 17]byte +var z963 [1 << 17]byte +var z964 [1 << 17]byte +var z965 [1 << 17]byte +var z966 [1 << 17]byte +var z967 [1 << 17]byte +var z968 [1 << 17]byte +var z969 [1 << 17]byte +var z970 [1 << 17]byte +var z971 [1 << 17]byte +var z972 [1 << 17]byte +var z973 [1 << 17]byte +var z974 [1 << 17]byte +var z975 [1 << 17]byte +var z976 [1 << 17]byte +var z977 [1 << 17]byte +var z978 [1 << 17]byte +var z979 [1 << 17]byte +var z980 [1 << 17]byte +var z981 [1 << 17]byte +var z982 [1 << 17]byte +var z983 [1 << 17]byte +var z984 [1 << 17]byte +var z985 [1 << 17]byte +var z986 [1 << 17]byte +var z987 [1 << 17]byte +var z988 [1 << 17]byte +var z989 [1 << 17]byte +var z990 [1 << 17]byte +var z991 [1 << 17]byte +var z992 [1 << 17]byte +var z993 [1 << 17]byte +var z994 [1 << 17]byte +var z995 [1 << 17]byte +var z996 [1 << 17]byte +var z997 [1 << 17]byte +var z998 [1 << 17]byte +var z999 [1 << 17]byte +var z1000 [1 << 17]byte +var z1001 [1 << 17]byte +var z1002 [1 << 17]byte +var z1003 [1 << 17]byte +var z1004 [1 << 17]byte +var z1005 [1 << 17]byte +var z1006 [1 << 17]byte +var z1007 [1 << 17]byte +var z1008 [1 << 17]byte +var z1009 [1 << 17]byte +var z1010 [1 << 17]byte +var z1011 [1 << 17]byte +var z1012 [1 << 17]byte +var z1013 [1 << 17]byte +var z1014 [1 << 17]byte +var z1015 [1 << 17]byte +var z1016 [1 << 17]byte +var z1017 [1 << 17]byte +var z1018 [1 << 17]byte +var z1019 [1 << 17]byte +var z1020 [1 << 17]byte +var z1021 [1 << 17]byte +var z1022 [1 << 17]byte +var z1023 [1 << 17]byte +var z1024 [1 << 17]byte +var z1025 [1 << 17]byte +var z1026 [1 << 17]byte +var z1027 [1 << 17]byte +var z1028 [1 << 17]byte +var z1029 [1 << 17]byte +var z1030 [1 << 17]byte +var z1031 [1 << 17]byte +var z1032 [1 << 17]byte +var z1033 [1 << 17]byte +var z1034 [1 << 17]byte +var z1035 [1 << 17]byte +var z1036 [1 << 17]byte +var z1037 [1 << 17]byte +var z1038 [1 << 17]byte +var z1039 [1 << 17]byte +var z1040 [1 << 17]byte +var z1041 [1 << 17]byte +var z1042 [1 << 17]byte +var z1043 [1 << 17]byte +var z1044 [1 << 17]byte +var z1045 [1 << 17]byte +var z1046 [1 << 17]byte +var z1047 [1 << 17]byte +var z1048 [1 << 17]byte +var z1049 [1 << 17]byte +var z1050 [1 << 17]byte +var z1051 [1 << 17]byte +var z1052 [1 << 17]byte +var z1053 [1 << 17]byte +var z1054 [1 << 17]byte +var z1055 [1 << 17]byte +var z1056 [1 << 17]byte +var z1057 [1 << 17]byte +var z1058 [1 << 17]byte +var z1059 [1 << 17]byte +var z1060 [1 << 17]byte +var z1061 [1 << 17]byte +var z1062 [1 << 17]byte +var z1063 [1 << 17]byte +var z1064 [1 << 17]byte +var z1065 [1 << 17]byte +var z1066 [1 << 17]byte +var z1067 [1 << 17]byte +var z1068 [1 << 17]byte +var z1069 [1 << 17]byte +var z1070 [1 << 17]byte +var z1071 [1 << 17]byte +var z1072 [1 << 17]byte +var z1073 [1 << 17]byte +var z1074 [1 << 17]byte +var z1075 [1 << 17]byte +var z1076 [1 << 17]byte +var z1077 [1 << 17]byte +var z1078 [1 << 17]byte +var z1079 [1 << 17]byte +var z1080 [1 << 17]byte +var z1081 [1 << 17]byte +var z1082 [1 << 17]byte +var z1083 [1 << 17]byte +var z1084 [1 << 17]byte +var z1085 [1 << 17]byte +var z1086 [1 << 17]byte +var z1087 [1 << 17]byte +var z1088 [1 << 17]byte +var z1089 [1 << 17]byte +var z1090 [1 << 17]byte +var z1091 [1 << 17]byte +var z1092 [1 << 17]byte +var z1093 [1 << 17]byte +var z1094 [1 << 17]byte +var z1095 [1 << 17]byte +var z1096 [1 << 17]byte +var z1097 [1 << 17]byte +var z1098 [1 << 17]byte +var z1099 [1 << 17]byte +var z1100 [1 << 17]byte +var z1101 [1 << 17]byte +var z1102 [1 << 17]byte +var z1103 [1 << 17]byte +var z1104 [1 << 17]byte +var z1105 [1 << 17]byte +var z1106 [1 << 17]byte +var z1107 [1 << 17]byte +var z1108 [1 << 17]byte +var z1109 [1 << 17]byte +var z1110 [1 << 17]byte +var z1111 [1 << 17]byte +var z1112 [1 << 17]byte +var z1113 [1 << 17]byte +var z1114 [1 << 17]byte +var z1115 [1 << 17]byte +var z1116 [1 << 17]byte +var z1117 [1 << 17]byte +var z1118 [1 << 17]byte +var z1119 [1 << 17]byte +var z1120 [1 << 17]byte +var z1121 [1 << 17]byte +var z1122 [1 << 17]byte +var z1123 [1 << 17]byte +var z1124 [1 << 17]byte +var z1125 [1 << 17]byte +var z1126 [1 << 17]byte +var z1127 [1 << 17]byte +var z1128 [1 << 17]byte +var z1129 [1 << 17]byte +var z1130 [1 << 17]byte +var z1131 [1 << 17]byte +var z1132 [1 << 17]byte +var z1133 [1 << 17]byte +var z1134 [1 << 17]byte +var z1135 [1 << 17]byte +var z1136 [1 << 17]byte +var z1137 [1 << 17]byte +var z1138 [1 << 17]byte +var z1139 [1 << 17]byte +var z1140 [1 << 17]byte +var z1141 [1 << 17]byte +var z1142 [1 << 17]byte +var z1143 [1 << 17]byte +var z1144 [1 << 17]byte +var z1145 [1 << 17]byte +var z1146 [1 << 17]byte +var z1147 [1 << 17]byte +var z1148 [1 << 17]byte +var z1149 [1 << 17]byte +var z1150 [1 << 17]byte +var z1151 [1 << 17]byte +var z1152 [1 << 17]byte +var z1153 [1 << 17]byte +var z1154 [1 << 17]byte +var z1155 [1 << 17]byte +var z1156 [1 << 17]byte +var z1157 [1 << 17]byte +var z1158 [1 << 17]byte +var z1159 [1 << 17]byte +var z1160 [1 << 17]byte +var z1161 [1 << 17]byte +var z1162 [1 << 17]byte +var z1163 [1 << 17]byte +var z1164 [1 << 17]byte +var z1165 [1 << 17]byte +var z1166 [1 << 17]byte +var z1167 [1 << 17]byte +var z1168 [1 << 17]byte +var z1169 [1 << 17]byte +var z1170 [1 << 17]byte +var z1171 [1 << 17]byte +var z1172 [1 << 17]byte +var z1173 [1 << 17]byte +var z1174 [1 << 17]byte +var z1175 [1 << 17]byte +var z1176 [1 << 17]byte +var z1177 [1 << 17]byte +var z1178 [1 << 17]byte +var z1179 [1 << 17]byte +var z1180 [1 << 17]byte +var z1181 [1 << 17]byte +var z1182 [1 << 17]byte +var z1183 [1 << 17]byte +var z1184 [1 << 17]byte +var z1185 [1 << 17]byte +var z1186 [1 << 17]byte +var z1187 [1 << 17]byte +var z1188 [1 << 17]byte +var z1189 [1 << 17]byte +var z1190 [1 << 17]byte +var z1191 [1 << 17]byte +var z1192 [1 << 17]byte +var z1193 [1 << 17]byte +var z1194 [1 << 17]byte +var z1195 [1 << 17]byte +var z1196 [1 << 17]byte +var z1197 [1 << 17]byte +var z1198 [1 << 17]byte +var z1199 [1 << 17]byte +var z1200 [1 << 17]byte +var z1201 [1 << 17]byte +var z1202 [1 << 17]byte +var z1203 [1 << 17]byte +var z1204 [1 << 17]byte +var z1205 [1 << 17]byte +var z1206 [1 << 17]byte +var z1207 [1 << 17]byte +var z1208 [1 << 17]byte +var z1209 [1 << 17]byte +var z1210 [1 << 17]byte +var z1211 [1 << 17]byte +var z1212 [1 << 17]byte +var z1213 [1 << 17]byte +var z1214 [1 << 17]byte +var z1215 [1 << 17]byte +var z1216 [1 << 17]byte +var z1217 [1 << 17]byte +var z1218 [1 << 17]byte +var z1219 [1 << 17]byte +var z1220 [1 << 17]byte +var z1221 [1 << 17]byte +var z1222 [1 << 17]byte +var z1223 [1 << 17]byte +var z1224 [1 << 17]byte +var z1225 [1 << 17]byte +var z1226 [1 << 17]byte +var z1227 [1 << 17]byte +var z1228 [1 << 17]byte +var z1229 [1 << 17]byte +var z1230 [1 << 17]byte +var z1231 [1 << 17]byte +var z1232 [1 << 17]byte +var z1233 [1 << 17]byte +var z1234 [1 << 17]byte +var z1235 [1 << 17]byte +var z1236 [1 << 17]byte +var z1237 [1 << 17]byte +var z1238 [1 << 17]byte +var z1239 [1 << 17]byte +var z1240 [1 << 17]byte +var z1241 [1 << 17]byte +var z1242 [1 << 17]byte +var z1243 [1 << 17]byte +var z1244 [1 << 17]byte +var z1245 [1 << 17]byte +var z1246 [1 << 17]byte +var z1247 [1 << 17]byte +var z1248 [1 << 17]byte +var z1249 [1 << 17]byte +var z1250 [1 << 17]byte +var z1251 [1 << 17]byte +var z1252 [1 << 17]byte +var z1253 [1 << 17]byte +var z1254 [1 << 17]byte +var z1255 [1 << 17]byte +var z1256 [1 << 17]byte +var z1257 [1 << 17]byte +var z1258 [1 << 17]byte +var z1259 [1 << 17]byte +var z1260 [1 << 17]byte +var z1261 [1 << 17]byte +var z1262 [1 << 17]byte +var z1263 [1 << 17]byte +var z1264 [1 << 17]byte +var z1265 [1 << 17]byte +var z1266 [1 << 17]byte +var z1267 [1 << 17]byte +var z1268 [1 << 17]byte +var z1269 [1 << 17]byte +var z1270 [1 << 17]byte +var z1271 [1 << 17]byte +var z1272 [1 << 17]byte +var z1273 [1 << 17]byte +var z1274 [1 << 17]byte +var z1275 [1 << 17]byte +var z1276 [1 << 17]byte +var z1277 [1 << 17]byte +var z1278 [1 << 17]byte +var z1279 [1 << 17]byte +var z1280 [1 << 17]byte +var z1281 [1 << 17]byte +var z1282 [1 << 17]byte +var z1283 [1 << 17]byte +var z1284 [1 << 17]byte +var z1285 [1 << 17]byte +var z1286 [1 << 17]byte +var z1287 [1 << 17]byte +var z1288 [1 << 17]byte +var z1289 [1 << 17]byte +var z1290 [1 << 17]byte +var z1291 [1 << 17]byte +var z1292 [1 << 17]byte +var z1293 [1 << 17]byte +var z1294 [1 << 17]byte +var z1295 [1 << 17]byte +var z1296 [1 << 17]byte +var z1297 [1 << 17]byte +var z1298 [1 << 17]byte +var z1299 [1 << 17]byte +var z1300 [1 << 17]byte +var z1301 [1 << 17]byte +var z1302 [1 << 17]byte +var z1303 [1 << 17]byte +var z1304 [1 << 17]byte +var z1305 [1 << 17]byte +var z1306 [1 << 17]byte +var z1307 [1 << 17]byte +var z1308 [1 << 17]byte +var z1309 [1 << 17]byte +var z1310 [1 << 17]byte +var z1311 [1 << 17]byte +var z1312 [1 << 17]byte +var z1313 [1 << 17]byte +var z1314 [1 << 17]byte +var z1315 [1 << 17]byte +var z1316 [1 << 17]byte +var z1317 [1 << 17]byte +var z1318 [1 << 17]byte +var z1319 [1 << 17]byte +var z1320 [1 << 17]byte +var z1321 [1 << 17]byte +var z1322 [1 << 17]byte +var z1323 [1 << 17]byte +var z1324 [1 << 17]byte +var z1325 [1 << 17]byte +var z1326 [1 << 17]byte +var z1327 [1 << 17]byte +var z1328 [1 << 17]byte +var z1329 [1 << 17]byte +var z1330 [1 << 17]byte +var z1331 [1 << 17]byte +var z1332 [1 << 17]byte +var z1333 [1 << 17]byte +var z1334 [1 << 17]byte +var z1335 [1 << 17]byte +var z1336 [1 << 17]byte +var z1337 [1 << 17]byte +var z1338 [1 << 17]byte +var z1339 [1 << 17]byte +var z1340 [1 << 17]byte +var z1341 [1 << 17]byte +var z1342 [1 << 17]byte +var z1343 [1 << 17]byte +var z1344 [1 << 17]byte +var z1345 [1 << 17]byte +var z1346 [1 << 17]byte +var z1347 [1 << 17]byte +var z1348 [1 << 17]byte +var z1349 [1 << 17]byte +var z1350 [1 << 17]byte +var z1351 [1 << 17]byte +var z1352 [1 << 17]byte +var z1353 [1 << 17]byte +var z1354 [1 << 17]byte +var z1355 [1 << 17]byte +var z1356 [1 << 17]byte +var z1357 [1 << 17]byte +var z1358 [1 << 17]byte +var z1359 [1 << 17]byte +var z1360 [1 << 17]byte +var z1361 [1 << 17]byte +var z1362 [1 << 17]byte +var z1363 [1 << 17]byte +var z1364 [1 << 17]byte +var z1365 [1 << 17]byte +var z1366 [1 << 17]byte +var z1367 [1 << 17]byte +var z1368 [1 << 17]byte +var z1369 [1 << 17]byte +var z1370 [1 << 17]byte +var z1371 [1 << 17]byte +var z1372 [1 << 17]byte +var z1373 [1 << 17]byte +var z1374 [1 << 17]byte +var z1375 [1 << 17]byte +var z1376 [1 << 17]byte +var z1377 [1 << 17]byte +var z1378 [1 << 17]byte +var z1379 [1 << 17]byte +var z1380 [1 << 17]byte +var z1381 [1 << 17]byte +var z1382 [1 << 17]byte +var z1383 [1 << 17]byte +var z1384 [1 << 17]byte +var z1385 [1 << 17]byte +var z1386 [1 << 17]byte +var z1387 [1 << 17]byte +var z1388 [1 << 17]byte +var z1389 [1 << 17]byte +var z1390 [1 << 17]byte +var z1391 [1 << 17]byte +var z1392 [1 << 17]byte +var z1393 [1 << 17]byte +var z1394 [1 << 17]byte +var z1395 [1 << 17]byte +var z1396 [1 << 17]byte +var z1397 [1 << 17]byte +var z1398 [1 << 17]byte +var z1399 [1 << 17]byte +var z1400 [1 << 17]byte +var z1401 [1 << 17]byte +var z1402 [1 << 17]byte +var z1403 [1 << 17]byte +var z1404 [1 << 17]byte +var z1405 [1 << 17]byte +var z1406 [1 << 17]byte +var z1407 [1 << 17]byte +var z1408 [1 << 17]byte +var z1409 [1 << 17]byte +var z1410 [1 << 17]byte +var z1411 [1 << 17]byte +var z1412 [1 << 17]byte +var z1413 [1 << 17]byte +var z1414 [1 << 17]byte +var z1415 [1 << 17]byte +var z1416 [1 << 17]byte +var z1417 [1 << 17]byte +var z1418 [1 << 17]byte +var z1419 [1 << 17]byte +var z1420 [1 << 17]byte +var z1421 [1 << 17]byte +var z1422 [1 << 17]byte +var z1423 [1 << 17]byte +var z1424 [1 << 17]byte +var z1425 [1 << 17]byte +var z1426 [1 << 17]byte +var z1427 [1 << 17]byte +var z1428 [1 << 17]byte +var z1429 [1 << 17]byte +var z1430 [1 << 17]byte +var z1431 [1 << 17]byte +var z1432 [1 << 17]byte +var z1433 [1 << 17]byte +var z1434 [1 << 17]byte +var z1435 [1 << 17]byte +var z1436 [1 << 17]byte +var z1437 [1 << 17]byte +var z1438 [1 << 17]byte +var z1439 [1 << 17]byte +var z1440 [1 << 17]byte +var z1441 [1 << 17]byte +var z1442 [1 << 17]byte +var z1443 [1 << 17]byte +var z1444 [1 << 17]byte +var z1445 [1 << 17]byte +var z1446 [1 << 17]byte +var z1447 [1 << 17]byte +var z1448 [1 << 17]byte +var z1449 [1 << 17]byte +var z1450 [1 << 17]byte +var z1451 [1 << 17]byte +var z1452 [1 << 17]byte +var z1453 [1 << 17]byte +var z1454 [1 << 17]byte +var z1455 [1 << 17]byte +var z1456 [1 << 17]byte +var z1457 [1 << 17]byte +var z1458 [1 << 17]byte +var z1459 [1 << 17]byte +var z1460 [1 << 17]byte +var z1461 [1 << 17]byte +var z1462 [1 << 17]byte +var z1463 [1 << 17]byte +var z1464 [1 << 17]byte +var z1465 [1 << 17]byte +var z1466 [1 << 17]byte +var z1467 [1 << 17]byte +var z1468 [1 << 17]byte +var z1469 [1 << 17]byte +var z1470 [1 << 17]byte +var z1471 [1 << 17]byte +var z1472 [1 << 17]byte +var z1473 [1 << 17]byte +var z1474 [1 << 17]byte +var z1475 [1 << 17]byte +var z1476 [1 << 17]byte +var z1477 [1 << 17]byte +var z1478 [1 << 17]byte +var z1479 [1 << 17]byte +var z1480 [1 << 17]byte +var z1481 [1 << 17]byte +var z1482 [1 << 17]byte +var z1483 [1 << 17]byte +var z1484 [1 << 17]byte +var z1485 [1 << 17]byte +var z1486 [1 << 17]byte +var z1487 [1 << 17]byte +var z1488 [1 << 17]byte +var z1489 [1 << 17]byte +var z1490 [1 << 17]byte +var z1491 [1 << 17]byte +var z1492 [1 << 17]byte +var z1493 [1 << 17]byte +var z1494 [1 << 17]byte +var z1495 [1 << 17]byte +var z1496 [1 << 17]byte +var z1497 [1 << 17]byte +var z1498 [1 << 17]byte +var z1499 [1 << 17]byte +var z1500 [1 << 17]byte +var z1501 [1 << 17]byte +var z1502 [1 << 17]byte +var z1503 [1 << 17]byte +var z1504 [1 << 17]byte +var z1505 [1 << 17]byte +var z1506 [1 << 17]byte +var z1507 [1 << 17]byte +var z1508 [1 << 17]byte +var z1509 [1 << 17]byte +var z1510 [1 << 17]byte +var z1511 [1 << 17]byte +var z1512 [1 << 17]byte +var z1513 [1 << 17]byte +var z1514 [1 << 17]byte +var z1515 [1 << 17]byte +var z1516 [1 << 17]byte +var z1517 [1 << 17]byte +var z1518 [1 << 17]byte +var z1519 [1 << 17]byte +var z1520 [1 << 17]byte +var z1521 [1 << 17]byte +var z1522 [1 << 17]byte +var z1523 [1 << 17]byte +var z1524 [1 << 17]byte +var z1525 [1 << 17]byte +var z1526 [1 << 17]byte +var z1527 [1 << 17]byte +var z1528 [1 << 17]byte +var z1529 [1 << 17]byte +var z1530 [1 << 17]byte +var z1531 [1 << 17]byte +var z1532 [1 << 17]byte +var z1533 [1 << 17]byte +var z1534 [1 << 17]byte +var z1535 [1 << 17]byte +var z1536 [1 << 17]byte +var z1537 [1 << 17]byte +var z1538 [1 << 17]byte +var z1539 [1 << 17]byte +var z1540 [1 << 17]byte +var z1541 [1 << 17]byte +var z1542 [1 << 17]byte +var z1543 [1 << 17]byte +var z1544 [1 << 17]byte +var z1545 [1 << 17]byte +var z1546 [1 << 17]byte +var z1547 [1 << 17]byte +var z1548 [1 << 17]byte +var z1549 [1 << 17]byte +var z1550 [1 << 17]byte +var z1551 [1 << 17]byte +var z1552 [1 << 17]byte +var z1553 [1 << 17]byte +var z1554 [1 << 17]byte +var z1555 [1 << 17]byte +var z1556 [1 << 17]byte +var z1557 [1 << 17]byte +var z1558 [1 << 17]byte +var z1559 [1 << 17]byte +var z1560 [1 << 17]byte +var z1561 [1 << 17]byte +var z1562 [1 << 17]byte +var z1563 [1 << 17]byte +var z1564 [1 << 17]byte +var z1565 [1 << 17]byte +var z1566 [1 << 17]byte +var z1567 [1 << 17]byte +var z1568 [1 << 17]byte +var z1569 [1 << 17]byte +var z1570 [1 << 17]byte +var z1571 [1 << 17]byte +var z1572 [1 << 17]byte +var z1573 [1 << 17]byte +var z1574 [1 << 17]byte +var z1575 [1 << 17]byte +var z1576 [1 << 17]byte +var z1577 [1 << 17]byte +var z1578 [1 << 17]byte +var z1579 [1 << 17]byte +var z1580 [1 << 17]byte +var z1581 [1 << 17]byte +var z1582 [1 << 17]byte +var z1583 [1 << 17]byte +var z1584 [1 << 17]byte +var z1585 [1 << 17]byte +var z1586 [1 << 17]byte +var z1587 [1 << 17]byte +var z1588 [1 << 17]byte +var z1589 [1 << 17]byte +var z1590 [1 << 17]byte +var z1591 [1 << 17]byte +var z1592 [1 << 17]byte +var z1593 [1 << 17]byte +var z1594 [1 << 17]byte +var z1595 [1 << 17]byte +var z1596 [1 << 17]byte +var z1597 [1 << 17]byte +var z1598 [1 << 17]byte +var z1599 [1 << 17]byte +var z1600 [1 << 17]byte +var z1601 [1 << 17]byte +var z1602 [1 << 17]byte +var z1603 [1 << 17]byte +var z1604 [1 << 17]byte +var z1605 [1 << 17]byte +var z1606 [1 << 17]byte +var z1607 [1 << 17]byte +var z1608 [1 << 17]byte +var z1609 [1 << 17]byte +var z1610 [1 << 17]byte +var z1611 [1 << 17]byte +var z1612 [1 << 17]byte +var z1613 [1 << 17]byte +var z1614 [1 << 17]byte +var z1615 [1 << 17]byte +var z1616 [1 << 17]byte +var z1617 [1 << 17]byte +var z1618 [1 << 17]byte +var z1619 [1 << 17]byte +var z1620 [1 << 17]byte +var z1621 [1 << 17]byte +var z1622 [1 << 17]byte +var z1623 [1 << 17]byte +var z1624 [1 << 17]byte +var z1625 [1 << 17]byte +var z1626 [1 << 17]byte +var z1627 [1 << 17]byte +var z1628 [1 << 17]byte +var z1629 [1 << 17]byte +var z1630 [1 << 17]byte +var z1631 [1 << 17]byte +var z1632 [1 << 17]byte +var z1633 [1 << 17]byte +var z1634 [1 << 17]byte +var z1635 [1 << 17]byte +var z1636 [1 << 17]byte +var z1637 [1 << 17]byte +var z1638 [1 << 17]byte +var z1639 [1 << 17]byte +var z1640 [1 << 17]byte +var z1641 [1 << 17]byte +var z1642 [1 << 17]byte +var z1643 [1 << 17]byte +var z1644 [1 << 17]byte +var z1645 [1 << 17]byte +var z1646 [1 << 17]byte +var z1647 [1 << 17]byte +var z1648 [1 << 17]byte +var z1649 [1 << 17]byte +var z1650 [1 << 17]byte +var z1651 [1 << 17]byte +var z1652 [1 << 17]byte +var z1653 [1 << 17]byte +var z1654 [1 << 17]byte +var z1655 [1 << 17]byte +var z1656 [1 << 17]byte +var z1657 [1 << 17]byte +var z1658 [1 << 17]byte +var z1659 [1 << 17]byte +var z1660 [1 << 17]byte +var z1661 [1 << 17]byte +var z1662 [1 << 17]byte +var z1663 [1 << 17]byte +var z1664 [1 << 17]byte +var z1665 [1 << 17]byte +var z1666 [1 << 17]byte +var z1667 [1 << 17]byte +var z1668 [1 << 17]byte +var z1669 [1 << 17]byte +var z1670 [1 << 17]byte +var z1671 [1 << 17]byte +var z1672 [1 << 17]byte +var z1673 [1 << 17]byte +var z1674 [1 << 17]byte +var z1675 [1 << 17]byte +var z1676 [1 << 17]byte +var z1677 [1 << 17]byte +var z1678 [1 << 17]byte +var z1679 [1 << 17]byte +var z1680 [1 << 17]byte +var z1681 [1 << 17]byte +var z1682 [1 << 17]byte +var z1683 [1 << 17]byte +var z1684 [1 << 17]byte +var z1685 [1 << 17]byte +var z1686 [1 << 17]byte +var z1687 [1 << 17]byte +var z1688 [1 << 17]byte +var z1689 [1 << 17]byte +var z1690 [1 << 17]byte +var z1691 [1 << 17]byte +var z1692 [1 << 17]byte +var z1693 [1 << 17]byte +var z1694 [1 << 17]byte +var z1695 [1 << 17]byte +var z1696 [1 << 17]byte +var z1697 [1 << 17]byte +var z1698 [1 << 17]byte +var z1699 [1 << 17]byte +var z1700 [1 << 17]byte +var z1701 [1 << 17]byte +var z1702 [1 << 17]byte +var z1703 [1 << 17]byte +var z1704 [1 << 17]byte +var z1705 [1 << 17]byte +var z1706 [1 << 17]byte +var z1707 [1 << 17]byte +var z1708 [1 << 17]byte +var z1709 [1 << 17]byte +var z1710 [1 << 17]byte +var z1711 [1 << 17]byte +var z1712 [1 << 17]byte +var z1713 [1 << 17]byte +var z1714 [1 << 17]byte +var z1715 [1 << 17]byte +var z1716 [1 << 17]byte +var z1717 [1 << 17]byte +var z1718 [1 << 17]byte +var z1719 [1 << 17]byte +var z1720 [1 << 17]byte +var z1721 [1 << 17]byte +var z1722 [1 << 17]byte +var z1723 [1 << 17]byte +var z1724 [1 << 17]byte +var z1725 [1 << 17]byte +var z1726 [1 << 17]byte +var z1727 [1 << 17]byte +var z1728 [1 << 17]byte +var z1729 [1 << 17]byte +var z1730 [1 << 17]byte +var z1731 [1 << 17]byte +var z1732 [1 << 17]byte +var z1733 [1 << 17]byte +var z1734 [1 << 17]byte +var z1735 [1 << 17]byte +var z1736 [1 << 17]byte +var z1737 [1 << 17]byte +var z1738 [1 << 17]byte +var z1739 [1 << 17]byte +var z1740 [1 << 17]byte +var z1741 [1 << 17]byte +var z1742 [1 << 17]byte +var z1743 [1 << 17]byte +var z1744 [1 << 17]byte +var z1745 [1 << 17]byte +var z1746 [1 << 17]byte +var z1747 [1 << 17]byte +var z1748 [1 << 17]byte +var z1749 [1 << 17]byte +var z1750 [1 << 17]byte +var z1751 [1 << 17]byte +var z1752 [1 << 17]byte +var z1753 [1 << 17]byte +var z1754 [1 << 17]byte +var z1755 [1 << 17]byte +var z1756 [1 << 17]byte +var z1757 [1 << 17]byte +var z1758 [1 << 17]byte +var z1759 [1 << 17]byte +var z1760 [1 << 17]byte +var z1761 [1 << 17]byte +var z1762 [1 << 17]byte +var z1763 [1 << 17]byte +var z1764 [1 << 17]byte +var z1765 [1 << 17]byte +var z1766 [1 << 17]byte +var z1767 [1 << 17]byte +var z1768 [1 << 17]byte +var z1769 [1 << 17]byte +var z1770 [1 << 17]byte +var z1771 [1 << 17]byte +var z1772 [1 << 17]byte +var z1773 [1 << 17]byte +var z1774 [1 << 17]byte +var z1775 [1 << 17]byte +var z1776 [1 << 17]byte +var z1777 [1 << 17]byte +var z1778 [1 << 17]byte +var z1779 [1 << 17]byte +var z1780 [1 << 17]byte +var z1781 [1 << 17]byte +var z1782 [1 << 17]byte +var z1783 [1 << 17]byte +var z1784 [1 << 17]byte +var z1785 [1 << 17]byte +var z1786 [1 << 17]byte +var z1787 [1 << 17]byte +var z1788 [1 << 17]byte +var z1789 [1 << 17]byte +var z1790 [1 << 17]byte +var z1791 [1 << 17]byte +var z1792 [1 << 17]byte +var z1793 [1 << 17]byte +var z1794 [1 << 17]byte +var z1795 [1 << 17]byte +var z1796 [1 << 17]byte +var z1797 [1 << 17]byte +var z1798 [1 << 17]byte +var z1799 [1 << 17]byte +var z1800 [1 << 17]byte +var z1801 [1 << 17]byte +var z1802 [1 << 17]byte +var z1803 [1 << 17]byte +var z1804 [1 << 17]byte +var z1805 [1 << 17]byte +var z1806 [1 << 17]byte +var z1807 [1 << 17]byte +var z1808 [1 << 17]byte +var z1809 [1 << 17]byte +var z1810 [1 << 17]byte +var z1811 [1 << 17]byte +var z1812 [1 << 17]byte +var z1813 [1 << 17]byte +var z1814 [1 << 17]byte +var z1815 [1 << 17]byte +var z1816 [1 << 17]byte +var z1817 [1 << 17]byte +var z1818 [1 << 17]byte +var z1819 [1 << 17]byte +var z1820 [1 << 17]byte +var z1821 [1 << 17]byte +var z1822 [1 << 17]byte +var z1823 [1 << 17]byte +var z1824 [1 << 17]byte +var z1825 [1 << 17]byte +var z1826 [1 << 17]byte +var z1827 [1 << 17]byte +var z1828 [1 << 17]byte +var z1829 [1 << 17]byte +var z1830 [1 << 17]byte +var z1831 [1 << 17]byte +var z1832 [1 << 17]byte +var z1833 [1 << 17]byte +var z1834 [1 << 17]byte +var z1835 [1 << 17]byte +var z1836 [1 << 17]byte +var z1837 [1 << 17]byte +var z1838 [1 << 17]byte +var z1839 [1 << 17]byte +var z1840 [1 << 17]byte +var z1841 [1 << 17]byte +var z1842 [1 << 17]byte +var z1843 [1 << 17]byte +var z1844 [1 << 17]byte +var z1845 [1 << 17]byte +var z1846 [1 << 17]byte +var z1847 [1 << 17]byte +var z1848 [1 << 17]byte +var z1849 [1 << 17]byte +var z1850 [1 << 17]byte +var z1851 [1 << 17]byte +var z1852 [1 << 17]byte +var z1853 [1 << 17]byte +var z1854 [1 << 17]byte +var z1855 [1 << 17]byte +var z1856 [1 << 17]byte +var z1857 [1 << 17]byte +var z1858 [1 << 17]byte +var z1859 [1 << 17]byte +var z1860 [1 << 17]byte +var z1861 [1 << 17]byte +var z1862 [1 << 17]byte +var z1863 [1 << 17]byte +var z1864 [1 << 17]byte +var z1865 [1 << 17]byte +var z1866 [1 << 17]byte +var z1867 [1 << 17]byte +var z1868 [1 << 17]byte +var z1869 [1 << 17]byte +var z1870 [1 << 17]byte +var z1871 [1 << 17]byte +var z1872 [1 << 17]byte +var z1873 [1 << 17]byte +var z1874 [1 << 17]byte +var z1875 [1 << 17]byte +var z1876 [1 << 17]byte +var z1877 [1 << 17]byte +var z1878 [1 << 17]byte +var z1879 [1 << 17]byte +var z1880 [1 << 17]byte +var z1881 [1 << 17]byte +var z1882 [1 << 17]byte +var z1883 [1 << 17]byte +var z1884 [1 << 17]byte +var z1885 [1 << 17]byte +var z1886 [1 << 17]byte +var z1887 [1 << 17]byte +var z1888 [1 << 17]byte +var z1889 [1 << 17]byte +var z1890 [1 << 17]byte +var z1891 [1 << 17]byte +var z1892 [1 << 17]byte +var z1893 [1 << 17]byte +var z1894 [1 << 17]byte +var z1895 [1 << 17]byte +var z1896 [1 << 17]byte +var z1897 [1 << 17]byte +var z1898 [1 << 17]byte +var z1899 [1 << 17]byte +var z1900 [1 << 17]byte +var z1901 [1 << 17]byte +var z1902 [1 << 17]byte +var z1903 [1 << 17]byte +var z1904 [1 << 17]byte +var z1905 [1 << 17]byte +var z1906 [1 << 17]byte +var z1907 [1 << 17]byte +var z1908 [1 << 17]byte +var z1909 [1 << 17]byte +var z1910 [1 << 17]byte +var z1911 [1 << 17]byte +var z1912 [1 << 17]byte +var z1913 [1 << 17]byte +var z1914 [1 << 17]byte +var z1915 [1 << 17]byte +var z1916 [1 << 17]byte +var z1917 [1 << 17]byte +var z1918 [1 << 17]byte +var z1919 [1 << 17]byte +var z1920 [1 << 17]byte +var z1921 [1 << 17]byte +var z1922 [1 << 17]byte +var z1923 [1 << 17]byte +var z1924 [1 << 17]byte +var z1925 [1 << 17]byte +var z1926 [1 << 17]byte +var z1927 [1 << 17]byte +var z1928 [1 << 17]byte +var z1929 [1 << 17]byte +var z1930 [1 << 17]byte +var z1931 [1 << 17]byte +var z1932 [1 << 17]byte +var z1933 [1 << 17]byte +var z1934 [1 << 17]byte +var z1935 [1 << 17]byte +var z1936 [1 << 17]byte +var z1937 [1 << 17]byte +var z1938 [1 << 17]byte +var z1939 [1 << 17]byte +var z1940 [1 << 17]byte +var z1941 [1 << 17]byte +var z1942 [1 << 17]byte +var z1943 [1 << 17]byte +var z1944 [1 << 17]byte +var z1945 [1 << 17]byte +var z1946 [1 << 17]byte +var z1947 [1 << 17]byte +var z1948 [1 << 17]byte +var z1949 [1 << 17]byte +var z1950 [1 << 17]byte +var z1951 [1 << 17]byte +var z1952 [1 << 17]byte +var z1953 [1 << 17]byte +var z1954 [1 << 17]byte +var z1955 [1 << 17]byte +var z1956 [1 << 17]byte +var z1957 [1 << 17]byte +var z1958 [1 << 17]byte +var z1959 [1 << 17]byte +var z1960 [1 << 17]byte +var z1961 [1 << 17]byte +var z1962 [1 << 17]byte +var z1963 [1 << 17]byte +var z1964 [1 << 17]byte +var z1965 [1 << 17]byte +var z1966 [1 << 17]byte +var z1967 [1 << 17]byte +var z1968 [1 << 17]byte +var z1969 [1 << 17]byte +var z1970 [1 << 17]byte +var z1971 [1 << 17]byte +var z1972 [1 << 17]byte +var z1973 [1 << 17]byte +var z1974 [1 << 17]byte +var z1975 [1 << 17]byte +var z1976 [1 << 17]byte +var z1977 [1 << 17]byte +var z1978 [1 << 17]byte +var z1979 [1 << 17]byte +var z1980 [1 << 17]byte +var z1981 [1 << 17]byte +var z1982 [1 << 17]byte +var z1983 [1 << 17]byte +var z1984 [1 << 17]byte +var z1985 [1 << 17]byte +var z1986 [1 << 17]byte +var z1987 [1 << 17]byte +var z1988 [1 << 17]byte +var z1989 [1 << 17]byte +var z1990 [1 << 17]byte +var z1991 [1 << 17]byte +var z1992 [1 << 17]byte +var z1993 [1 << 17]byte +var z1994 [1 << 17]byte +var z1995 [1 << 17]byte +var z1996 [1 << 17]byte +var z1997 [1 << 17]byte +var z1998 [1 << 17]byte +var z1999 [1 << 17]byte +var z2000 [1 << 17]byte +var z2001 [1 << 17]byte +var z2002 [1 << 17]byte +var z2003 [1 << 17]byte +var z2004 [1 << 17]byte +var z2005 [1 << 17]byte +var z2006 [1 << 17]byte +var z2007 [1 << 17]byte +var z2008 [1 << 17]byte +var z2009 [1 << 17]byte +var z2010 [1 << 17]byte +var z2011 [1 << 17]byte +var z2012 [1 << 17]byte +var z2013 [1 << 17]byte +var z2014 [1 << 17]byte +var z2015 [1 << 17]byte +var z2016 [1 << 17]byte +var z2017 [1 << 17]byte +var z2018 [1 << 17]byte +var z2019 [1 << 17]byte +var z2020 [1 << 17]byte +var z2021 [1 << 17]byte +var z2022 [1 << 17]byte +var z2023 [1 << 17]byte +var z2024 [1 << 17]byte +var z2025 [1 << 17]byte +var z2026 [1 << 17]byte +var z2027 [1 << 17]byte +var z2028 [1 << 17]byte +var z2029 [1 << 17]byte +var z2030 [1 << 17]byte +var z2031 [1 << 17]byte +var z2032 [1 << 17]byte +var z2033 [1 << 17]byte +var z2034 [1 << 17]byte +var z2035 [1 << 17]byte +var z2036 [1 << 17]byte +var z2037 [1 << 17]byte +var z2038 [1 << 17]byte +var z2039 [1 << 17]byte +var z2040 [1 << 17]byte +var z2041 [1 << 17]byte +var z2042 [1 << 17]byte +var z2043 [1 << 17]byte +var z2044 [1 << 17]byte +var z2045 [1 << 17]byte +var z2046 [1 << 17]byte +var z2047 [1 << 17]byte +var z2048 [1 << 17]byte +var z2049 [1 << 17]byte +var z2050 [1 << 17]byte +var z2051 [1 << 17]byte +var z2052 [1 << 17]byte +var z2053 [1 << 17]byte +var z2054 [1 << 17]byte +var z2055 [1 << 17]byte +var z2056 [1 << 17]byte +var z2057 [1 << 17]byte +var z2058 [1 << 17]byte +var z2059 [1 << 17]byte +var z2060 [1 << 17]byte +var z2061 [1 << 17]byte +var z2062 [1 << 17]byte +var z2063 [1 << 17]byte +var z2064 [1 << 17]byte +var z2065 [1 << 17]byte +var z2066 [1 << 17]byte +var z2067 [1 << 17]byte +var z2068 [1 << 17]byte +var z2069 [1 << 17]byte +var z2070 [1 << 17]byte +var z2071 [1 << 17]byte +var z2072 [1 << 17]byte +var z2073 [1 << 17]byte +var z2074 [1 << 17]byte +var z2075 [1 << 17]byte +var z2076 [1 << 17]byte +var z2077 [1 << 17]byte +var z2078 [1 << 17]byte +var z2079 [1 << 17]byte +var z2080 [1 << 17]byte +var z2081 [1 << 17]byte +var z2082 [1 << 17]byte +var z2083 [1 << 17]byte +var z2084 [1 << 17]byte +var z2085 [1 << 17]byte +var z2086 [1 << 17]byte +var z2087 [1 << 17]byte +var z2088 [1 << 17]byte +var z2089 [1 << 17]byte +var z2090 [1 << 17]byte +var z2091 [1 << 17]byte +var z2092 [1 << 17]byte +var z2093 [1 << 17]byte +var z2094 [1 << 17]byte +var z2095 [1 << 17]byte +var z2096 [1 << 17]byte +var z2097 [1 << 17]byte +var z2098 [1 << 17]byte +var z2099 [1 << 17]byte +var z2100 [1 << 17]byte +var z2101 [1 << 17]byte +var z2102 [1 << 17]byte +var z2103 [1 << 17]byte +var z2104 [1 << 17]byte +var z2105 [1 << 17]byte +var z2106 [1 << 17]byte +var z2107 [1 << 17]byte +var z2108 [1 << 17]byte +var z2109 [1 << 17]byte +var z2110 [1 << 17]byte +var z2111 [1 << 17]byte +var z2112 [1 << 17]byte +var z2113 [1 << 17]byte +var z2114 [1 << 17]byte +var z2115 [1 << 17]byte +var z2116 [1 << 17]byte +var z2117 [1 << 17]byte +var z2118 [1 << 17]byte +var z2119 [1 << 17]byte +var z2120 [1 << 17]byte +var z2121 [1 << 17]byte +var z2122 [1 << 17]byte +var z2123 [1 << 17]byte +var z2124 [1 << 17]byte +var z2125 [1 << 17]byte +var z2126 [1 << 17]byte +var z2127 [1 << 17]byte +var z2128 [1 << 17]byte +var z2129 [1 << 17]byte +var z2130 [1 << 17]byte +var z2131 [1 << 17]byte +var z2132 [1 << 17]byte +var z2133 [1 << 17]byte +var z2134 [1 << 17]byte +var z2135 [1 << 17]byte +var z2136 [1 << 17]byte +var z2137 [1 << 17]byte +var z2138 [1 << 17]byte +var z2139 [1 << 17]byte +var z2140 [1 << 17]byte +var z2141 [1 << 17]byte +var z2142 [1 << 17]byte +var z2143 [1 << 17]byte +var z2144 [1 << 17]byte +var z2145 [1 << 17]byte +var z2146 [1 << 17]byte +var z2147 [1 << 17]byte +var z2148 [1 << 17]byte +var z2149 [1 << 17]byte +var z2150 [1 << 17]byte +var z2151 [1 << 17]byte +var z2152 [1 << 17]byte +var z2153 [1 << 17]byte +var z2154 [1 << 17]byte +var z2155 [1 << 17]byte +var z2156 [1 << 17]byte +var z2157 [1 << 17]byte +var z2158 [1 << 17]byte +var z2159 [1 << 17]byte +var z2160 [1 << 17]byte +var z2161 [1 << 17]byte +var z2162 [1 << 17]byte +var z2163 [1 << 17]byte +var z2164 [1 << 17]byte +var z2165 [1 << 17]byte +var z2166 [1 << 17]byte +var z2167 [1 << 17]byte +var z2168 [1 << 17]byte +var z2169 [1 << 17]byte +var z2170 [1 << 17]byte +var z2171 [1 << 17]byte +var z2172 [1 << 17]byte +var z2173 [1 << 17]byte +var z2174 [1 << 17]byte +var z2175 [1 << 17]byte +var z2176 [1 << 17]byte +var z2177 [1 << 17]byte +var z2178 [1 << 17]byte +var z2179 [1 << 17]byte +var z2180 [1 << 17]byte +var z2181 [1 << 17]byte +var z2182 [1 << 17]byte +var z2183 [1 << 17]byte +var z2184 [1 << 17]byte +var z2185 [1 << 17]byte +var z2186 [1 << 17]byte +var z2187 [1 << 17]byte +var z2188 [1 << 17]byte +var z2189 [1 << 17]byte +var z2190 [1 << 17]byte +var z2191 [1 << 17]byte +var z2192 [1 << 17]byte +var z2193 [1 << 17]byte +var z2194 [1 << 17]byte +var z2195 [1 << 17]byte +var z2196 [1 << 17]byte +var z2197 [1 << 17]byte +var z2198 [1 << 17]byte +var z2199 [1 << 17]byte +var z2200 [1 << 17]byte +var z2201 [1 << 17]byte +var z2202 [1 << 17]byte +var z2203 [1 << 17]byte +var z2204 [1 << 17]byte +var z2205 [1 << 17]byte +var z2206 [1 << 17]byte +var z2207 [1 << 17]byte +var z2208 [1 << 17]byte +var z2209 [1 << 17]byte +var z2210 [1 << 17]byte +var z2211 [1 << 17]byte +var z2212 [1 << 17]byte +var z2213 [1 << 17]byte +var z2214 [1 << 17]byte +var z2215 [1 << 17]byte +var z2216 [1 << 17]byte +var z2217 [1 << 17]byte +var z2218 [1 << 17]byte +var z2219 [1 << 17]byte +var z2220 [1 << 17]byte +var z2221 [1 << 17]byte +var z2222 [1 << 17]byte +var z2223 [1 << 17]byte +var z2224 [1 << 17]byte +var z2225 [1 << 17]byte +var z2226 [1 << 17]byte +var z2227 [1 << 17]byte +var z2228 [1 << 17]byte +var z2229 [1 << 17]byte +var z2230 [1 << 17]byte +var z2231 [1 << 17]byte +var z2232 [1 << 17]byte +var z2233 [1 << 17]byte +var z2234 [1 << 17]byte +var z2235 [1 << 17]byte +var z2236 [1 << 17]byte +var z2237 [1 << 17]byte +var z2238 [1 << 17]byte +var z2239 [1 << 17]byte +var z2240 [1 << 17]byte +var z2241 [1 << 17]byte +var z2242 [1 << 17]byte +var z2243 [1 << 17]byte +var z2244 [1 << 17]byte +var z2245 [1 << 17]byte +var z2246 [1 << 17]byte +var z2247 [1 << 17]byte +var z2248 [1 << 17]byte +var z2249 [1 << 17]byte +var z2250 [1 << 17]byte +var z2251 [1 << 17]byte +var z2252 [1 << 17]byte +var z2253 [1 << 17]byte +var z2254 [1 << 17]byte +var z2255 [1 << 17]byte +var z2256 [1 << 17]byte +var z2257 [1 << 17]byte +var z2258 [1 << 17]byte +var z2259 [1 << 17]byte +var z2260 [1 << 17]byte +var z2261 [1 << 17]byte +var z2262 [1 << 17]byte +var z2263 [1 << 17]byte +var z2264 [1 << 17]byte +var z2265 [1 << 17]byte +var z2266 [1 << 17]byte +var z2267 [1 << 17]byte +var z2268 [1 << 17]byte +var z2269 [1 << 17]byte +var z2270 [1 << 17]byte +var z2271 [1 << 17]byte +var z2272 [1 << 17]byte +var z2273 [1 << 17]byte +var z2274 [1 << 17]byte +var z2275 [1 << 17]byte +var z2276 [1 << 17]byte +var z2277 [1 << 17]byte +var z2278 [1 << 17]byte +var z2279 [1 << 17]byte +var z2280 [1 << 17]byte +var z2281 [1 << 17]byte +var z2282 [1 << 17]byte +var z2283 [1 << 17]byte +var z2284 [1 << 17]byte +var z2285 [1 << 17]byte +var z2286 [1 << 17]byte +var z2287 [1 << 17]byte +var z2288 [1 << 17]byte +var z2289 [1 << 17]byte +var z2290 [1 << 17]byte +var z2291 [1 << 17]byte +var z2292 [1 << 17]byte +var z2293 [1 << 17]byte +var z2294 [1 << 17]byte +var z2295 [1 << 17]byte +var z2296 [1 << 17]byte +var z2297 [1 << 17]byte +var z2298 [1 << 17]byte +var z2299 [1 << 17]byte +var z2300 [1 << 17]byte +var z2301 [1 << 17]byte +var z2302 [1 << 17]byte +var z2303 [1 << 17]byte +var z2304 [1 << 17]byte +var z2305 [1 << 17]byte +var z2306 [1 << 17]byte +var z2307 [1 << 17]byte +var z2308 [1 << 17]byte +var z2309 [1 << 17]byte +var z2310 [1 << 17]byte +var z2311 [1 << 17]byte +var z2312 [1 << 17]byte +var z2313 [1 << 17]byte +var z2314 [1 << 17]byte +var z2315 [1 << 17]byte +var z2316 [1 << 17]byte +var z2317 [1 << 17]byte +var z2318 [1 << 17]byte +var z2319 [1 << 17]byte +var z2320 [1 << 17]byte +var z2321 [1 << 17]byte +var z2322 [1 << 17]byte +var z2323 [1 << 17]byte +var z2324 [1 << 17]byte +var z2325 [1 << 17]byte +var z2326 [1 << 17]byte +var z2327 [1 << 17]byte +var z2328 [1 << 17]byte +var z2329 [1 << 17]byte +var z2330 [1 << 17]byte +var z2331 [1 << 17]byte +var z2332 [1 << 17]byte +var z2333 [1 << 17]byte +var z2334 [1 << 17]byte +var z2335 [1 << 17]byte +var z2336 [1 << 17]byte +var z2337 [1 << 17]byte +var z2338 [1 << 17]byte +var z2339 [1 << 17]byte +var z2340 [1 << 17]byte +var z2341 [1 << 17]byte +var z2342 [1 << 17]byte +var z2343 [1 << 17]byte +var z2344 [1 << 17]byte +var z2345 [1 << 17]byte +var z2346 [1 << 17]byte +var z2347 [1 << 17]byte +var z2348 [1 << 17]byte +var z2349 [1 << 17]byte +var z2350 [1 << 17]byte +var z2351 [1 << 17]byte +var z2352 [1 << 17]byte +var z2353 [1 << 17]byte +var z2354 [1 << 17]byte +var z2355 [1 << 17]byte +var z2356 [1 << 17]byte +var z2357 [1 << 17]byte +var z2358 [1 << 17]byte +var z2359 [1 << 17]byte +var z2360 [1 << 17]byte +var z2361 [1 << 17]byte +var z2362 [1 << 17]byte +var z2363 [1 << 17]byte +var z2364 [1 << 17]byte +var z2365 [1 << 17]byte +var z2366 [1 << 17]byte +var z2367 [1 << 17]byte +var z2368 [1 << 17]byte +var z2369 [1 << 17]byte +var z2370 [1 << 17]byte +var z2371 [1 << 17]byte +var z2372 [1 << 17]byte +var z2373 [1 << 17]byte +var z2374 [1 << 17]byte +var z2375 [1 << 17]byte +var z2376 [1 << 17]byte +var z2377 [1 << 17]byte +var z2378 [1 << 17]byte +var z2379 [1 << 17]byte +var z2380 [1 << 17]byte +var z2381 [1 << 17]byte +var z2382 [1 << 17]byte +var z2383 [1 << 17]byte +var z2384 [1 << 17]byte +var z2385 [1 << 17]byte +var z2386 [1 << 17]byte +var z2387 [1 << 17]byte +var z2388 [1 << 17]byte +var z2389 [1 << 17]byte +var z2390 [1 << 17]byte +var z2391 [1 << 17]byte +var z2392 [1 << 17]byte +var z2393 [1 << 17]byte +var z2394 [1 << 17]byte +var z2395 [1 << 17]byte +var z2396 [1 << 17]byte +var z2397 [1 << 17]byte +var z2398 [1 << 17]byte +var z2399 [1 << 17]byte +var z2400 [1 << 17]byte +var z2401 [1 << 17]byte +var z2402 [1 << 17]byte +var z2403 [1 << 17]byte +var z2404 [1 << 17]byte +var z2405 [1 << 17]byte +var z2406 [1 << 17]byte +var z2407 [1 << 17]byte +var z2408 [1 << 17]byte +var z2409 [1 << 17]byte +var z2410 [1 << 17]byte +var z2411 [1 << 17]byte +var z2412 [1 << 17]byte +var z2413 [1 << 17]byte +var z2414 [1 << 17]byte +var z2415 [1 << 17]byte +var z2416 [1 << 17]byte +var z2417 [1 << 17]byte +var z2418 [1 << 17]byte +var z2419 [1 << 17]byte +var z2420 [1 << 17]byte +var z2421 [1 << 17]byte +var z2422 [1 << 17]byte +var z2423 [1 << 17]byte +var z2424 [1 << 17]byte +var z2425 [1 << 17]byte +var z2426 [1 << 17]byte +var z2427 [1 << 17]byte +var z2428 [1 << 17]byte +var z2429 [1 << 17]byte +var z2430 [1 << 17]byte +var z2431 [1 << 17]byte +var z2432 [1 << 17]byte +var z2433 [1 << 17]byte +var z2434 [1 << 17]byte +var z2435 [1 << 17]byte +var z2436 [1 << 17]byte +var z2437 [1 << 17]byte +var z2438 [1 << 17]byte +var z2439 [1 << 17]byte +var z2440 [1 << 17]byte +var z2441 [1 << 17]byte +var z2442 [1 << 17]byte +var z2443 [1 << 17]byte +var z2444 [1 << 17]byte +var z2445 [1 << 17]byte +var z2446 [1 << 17]byte +var z2447 [1 << 17]byte +var z2448 [1 << 17]byte +var z2449 [1 << 17]byte +var z2450 [1 << 17]byte +var z2451 [1 << 17]byte +var z2452 [1 << 17]byte +var z2453 [1 << 17]byte +var z2454 [1 << 17]byte +var z2455 [1 << 17]byte +var z2456 [1 << 17]byte +var z2457 [1 << 17]byte +var z2458 [1 << 17]byte +var z2459 [1 << 17]byte +var z2460 [1 << 17]byte +var z2461 [1 << 17]byte +var z2462 [1 << 17]byte +var z2463 [1 << 17]byte +var z2464 [1 << 17]byte +var z2465 [1 << 17]byte +var z2466 [1 << 17]byte +var z2467 [1 << 17]byte +var z2468 [1 << 17]byte +var z2469 [1 << 17]byte +var z2470 [1 << 17]byte +var z2471 [1 << 17]byte +var z2472 [1 << 17]byte +var z2473 [1 << 17]byte +var z2474 [1 << 17]byte +var z2475 [1 << 17]byte +var z2476 [1 << 17]byte +var z2477 [1 << 17]byte +var z2478 [1 << 17]byte +var z2479 [1 << 17]byte +var z2480 [1 << 17]byte +var z2481 [1 << 17]byte +var z2482 [1 << 17]byte +var z2483 [1 << 17]byte +var z2484 [1 << 17]byte +var z2485 [1 << 17]byte +var z2486 [1 << 17]byte +var z2487 [1 << 17]byte +var z2488 [1 << 17]byte +var z2489 [1 << 17]byte +var z2490 [1 << 17]byte +var z2491 [1 << 17]byte +var z2492 [1 << 17]byte +var z2493 [1 << 17]byte +var z2494 [1 << 17]byte +var z2495 [1 << 17]byte +var z2496 [1 << 17]byte +var z2497 [1 << 17]byte +var z2498 [1 << 17]byte +var z2499 [1 << 17]byte +var z2500 [1 << 17]byte +var z2501 [1 << 17]byte +var z2502 [1 << 17]byte +var z2503 [1 << 17]byte +var z2504 [1 << 17]byte +var z2505 [1 << 17]byte +var z2506 [1 << 17]byte +var z2507 [1 << 17]byte +var z2508 [1 << 17]byte +var z2509 [1 << 17]byte +var z2510 [1 << 17]byte +var z2511 [1 << 17]byte +var z2512 [1 << 17]byte +var z2513 [1 << 17]byte +var z2514 [1 << 17]byte +var z2515 [1 << 17]byte +var z2516 [1 << 17]byte +var z2517 [1 << 17]byte +var z2518 [1 << 17]byte +var z2519 [1 << 17]byte +var z2520 [1 << 17]byte +var z2521 [1 << 17]byte +var z2522 [1 << 17]byte +var z2523 [1 << 17]byte +var z2524 [1 << 17]byte +var z2525 [1 << 17]byte +var z2526 [1 << 17]byte +var z2527 [1 << 17]byte +var z2528 [1 << 17]byte +var z2529 [1 << 17]byte +var z2530 [1 << 17]byte +var z2531 [1 << 17]byte +var z2532 [1 << 17]byte +var z2533 [1 << 17]byte +var z2534 [1 << 17]byte +var z2535 [1 << 17]byte +var z2536 [1 << 17]byte +var z2537 [1 << 17]byte +var z2538 [1 << 17]byte +var z2539 [1 << 17]byte +var z2540 [1 << 17]byte +var z2541 [1 << 17]byte +var z2542 [1 << 17]byte +var z2543 [1 << 17]byte +var z2544 [1 << 17]byte +var z2545 [1 << 17]byte +var z2546 [1 << 17]byte +var z2547 [1 << 17]byte +var z2548 [1 << 17]byte +var z2549 [1 << 17]byte +var z2550 [1 << 17]byte +var z2551 [1 << 17]byte +var z2552 [1 << 17]byte +var z2553 [1 << 17]byte +var z2554 [1 << 17]byte +var z2555 [1 << 17]byte +var z2556 [1 << 17]byte +var z2557 [1 << 17]byte +var z2558 [1 << 17]byte +var z2559 [1 << 17]byte +var z2560 [1 << 17]byte +var z2561 [1 << 17]byte +var z2562 [1 << 17]byte +var z2563 [1 << 17]byte +var z2564 [1 << 17]byte +var z2565 [1 << 17]byte +var z2566 [1 << 17]byte +var z2567 [1 << 17]byte +var z2568 [1 << 17]byte +var z2569 [1 << 17]byte +var z2570 [1 << 17]byte +var z2571 [1 << 17]byte +var z2572 [1 << 17]byte +var z2573 [1 << 17]byte +var z2574 [1 << 17]byte +var z2575 [1 << 17]byte +var z2576 [1 << 17]byte +var z2577 [1 << 17]byte +var z2578 [1 << 17]byte +var z2579 [1 << 17]byte +var z2580 [1 << 17]byte +var z2581 [1 << 17]byte +var z2582 [1 << 17]byte +var z2583 [1 << 17]byte +var z2584 [1 << 17]byte +var z2585 [1 << 17]byte +var z2586 [1 << 17]byte +var z2587 [1 << 17]byte +var z2588 [1 << 17]byte +var z2589 [1 << 17]byte +var z2590 [1 << 17]byte +var z2591 [1 << 17]byte +var z2592 [1 << 17]byte +var z2593 [1 << 17]byte +var z2594 [1 << 17]byte +var z2595 [1 << 17]byte +var z2596 [1 << 17]byte +var z2597 [1 << 17]byte +var z2598 [1 << 17]byte +var z2599 [1 << 17]byte +var z2600 [1 << 17]byte +var z2601 [1 << 17]byte +var z2602 [1 << 17]byte +var z2603 [1 << 17]byte +var z2604 [1 << 17]byte +var z2605 [1 << 17]byte +var z2606 [1 << 17]byte +var z2607 [1 << 17]byte +var z2608 [1 << 17]byte +var z2609 [1 << 17]byte +var z2610 [1 << 17]byte +var z2611 [1 << 17]byte +var z2612 [1 << 17]byte +var z2613 [1 << 17]byte +var z2614 [1 << 17]byte +var z2615 [1 << 17]byte +var z2616 [1 << 17]byte +var z2617 [1 << 17]byte +var z2618 [1 << 17]byte +var z2619 [1 << 17]byte +var z2620 [1 << 17]byte +var z2621 [1 << 17]byte +var z2622 [1 << 17]byte +var z2623 [1 << 17]byte +var z2624 [1 << 17]byte +var z2625 [1 << 17]byte +var z2626 [1 << 17]byte +var z2627 [1 << 17]byte +var z2628 [1 << 17]byte +var z2629 [1 << 17]byte +var z2630 [1 << 17]byte +var z2631 [1 << 17]byte +var z2632 [1 << 17]byte +var z2633 [1 << 17]byte +var z2634 [1 << 17]byte +var z2635 [1 << 17]byte +var z2636 [1 << 17]byte +var z2637 [1 << 17]byte +var z2638 [1 << 17]byte +var z2639 [1 << 17]byte +var z2640 [1 << 17]byte +var z2641 [1 << 17]byte +var z2642 [1 << 17]byte +var z2643 [1 << 17]byte +var z2644 [1 << 17]byte +var z2645 [1 << 17]byte +var z2646 [1 << 17]byte +var z2647 [1 << 17]byte +var z2648 [1 << 17]byte +var z2649 [1 << 17]byte +var z2650 [1 << 17]byte +var z2651 [1 << 17]byte +var z2652 [1 << 17]byte +var z2653 [1 << 17]byte +var z2654 [1 << 17]byte +var z2655 [1 << 17]byte +var z2656 [1 << 17]byte +var z2657 [1 << 17]byte +var z2658 [1 << 17]byte +var z2659 [1 << 17]byte +var z2660 [1 << 17]byte +var z2661 [1 << 17]byte +var z2662 [1 << 17]byte +var z2663 [1 << 17]byte +var z2664 [1 << 17]byte +var z2665 [1 << 17]byte +var z2666 [1 << 17]byte +var z2667 [1 << 17]byte +var z2668 [1 << 17]byte +var z2669 [1 << 17]byte +var z2670 [1 << 17]byte +var z2671 [1 << 17]byte +var z2672 [1 << 17]byte +var z2673 [1 << 17]byte +var z2674 [1 << 17]byte +var z2675 [1 << 17]byte +var z2676 [1 << 17]byte +var z2677 [1 << 17]byte +var z2678 [1 << 17]byte +var z2679 [1 << 17]byte +var z2680 [1 << 17]byte +var z2681 [1 << 17]byte +var z2682 [1 << 17]byte +var z2683 [1 << 17]byte +var z2684 [1 << 17]byte +var z2685 [1 << 17]byte +var z2686 [1 << 17]byte +var z2687 [1 << 17]byte +var z2688 [1 << 17]byte +var z2689 [1 << 17]byte +var z2690 [1 << 17]byte +var z2691 [1 << 17]byte +var z2692 [1 << 17]byte +var z2693 [1 << 17]byte +var z2694 [1 << 17]byte +var z2695 [1 << 17]byte +var z2696 [1 << 17]byte +var z2697 [1 << 17]byte +var z2698 [1 << 17]byte +var z2699 [1 << 17]byte +var z2700 [1 << 17]byte +var z2701 [1 << 17]byte +var z2702 [1 << 17]byte +var z2703 [1 << 17]byte +var z2704 [1 << 17]byte +var z2705 [1 << 17]byte +var z2706 [1 << 17]byte +var z2707 [1 << 17]byte +var z2708 [1 << 17]byte +var z2709 [1 << 17]byte +var z2710 [1 << 17]byte +var z2711 [1 << 17]byte +var z2712 [1 << 17]byte +var z2713 [1 << 17]byte +var z2714 [1 << 17]byte +var z2715 [1 << 17]byte +var z2716 [1 << 17]byte +var z2717 [1 << 17]byte +var z2718 [1 << 17]byte +var z2719 [1 << 17]byte +var z2720 [1 << 17]byte +var z2721 [1 << 17]byte +var z2722 [1 << 17]byte +var z2723 [1 << 17]byte +var z2724 [1 << 17]byte +var z2725 [1 << 17]byte +var z2726 [1 << 17]byte +var z2727 [1 << 17]byte +var z2728 [1 << 17]byte +var z2729 [1 << 17]byte +var z2730 [1 << 17]byte +var z2731 [1 << 17]byte +var z2732 [1 << 17]byte +var z2733 [1 << 17]byte +var z2734 [1 << 17]byte +var z2735 [1 << 17]byte +var z2736 [1 << 17]byte +var z2737 [1 << 17]byte +var z2738 [1 << 17]byte +var z2739 [1 << 17]byte +var z2740 [1 << 17]byte +var z2741 [1 << 17]byte +var z2742 [1 << 17]byte +var z2743 [1 << 17]byte +var z2744 [1 << 17]byte +var z2745 [1 << 17]byte +var z2746 [1 << 17]byte +var z2747 [1 << 17]byte +var z2748 [1 << 17]byte +var z2749 [1 << 17]byte +var z2750 [1 << 17]byte +var z2751 [1 << 17]byte +var z2752 [1 << 17]byte +var z2753 [1 << 17]byte +var z2754 [1 << 17]byte +var z2755 [1 << 17]byte +var z2756 [1 << 17]byte +var z2757 [1 << 17]byte +var z2758 [1 << 17]byte +var z2759 [1 << 17]byte +var z2760 [1 << 17]byte +var z2761 [1 << 17]byte +var z2762 [1 << 17]byte +var z2763 [1 << 17]byte +var z2764 [1 << 17]byte +var z2765 [1 << 17]byte +var z2766 [1 << 17]byte +var z2767 [1 << 17]byte +var z2768 [1 << 17]byte +var z2769 [1 << 17]byte +var z2770 [1 << 17]byte +var z2771 [1 << 17]byte +var z2772 [1 << 17]byte +var z2773 [1 << 17]byte +var z2774 [1 << 17]byte +var z2775 [1 << 17]byte +var z2776 [1 << 17]byte +var z2777 [1 << 17]byte +var z2778 [1 << 17]byte +var z2779 [1 << 17]byte +var z2780 [1 << 17]byte +var z2781 [1 << 17]byte +var z2782 [1 << 17]byte +var z2783 [1 << 17]byte +var z2784 [1 << 17]byte +var z2785 [1 << 17]byte +var z2786 [1 << 17]byte +var z2787 [1 << 17]byte +var z2788 [1 << 17]byte +var z2789 [1 << 17]byte +var z2790 [1 << 17]byte +var z2791 [1 << 17]byte +var z2792 [1 << 17]byte +var z2793 [1 << 17]byte +var z2794 [1 << 17]byte +var z2795 [1 << 17]byte +var z2796 [1 << 17]byte +var z2797 [1 << 17]byte +var z2798 [1 << 17]byte +var z2799 [1 << 17]byte +var z2800 [1 << 17]byte +var z2801 [1 << 17]byte +var z2802 [1 << 17]byte +var z2803 [1 << 17]byte +var z2804 [1 << 17]byte +var z2805 [1 << 17]byte +var z2806 [1 << 17]byte +var z2807 [1 << 17]byte +var z2808 [1 << 17]byte +var z2809 [1 << 17]byte +var z2810 [1 << 17]byte +var z2811 [1 << 17]byte +var z2812 [1 << 17]byte +var z2813 [1 << 17]byte +var z2814 [1 << 17]byte +var z2815 [1 << 17]byte +var z2816 [1 << 17]byte +var z2817 [1 << 17]byte +var z2818 [1 << 17]byte +var z2819 [1 << 17]byte +var z2820 [1 << 17]byte +var z2821 [1 << 17]byte +var z2822 [1 << 17]byte +var z2823 [1 << 17]byte +var z2824 [1 << 17]byte +var z2825 [1 << 17]byte +var z2826 [1 << 17]byte +var z2827 [1 << 17]byte +var z2828 [1 << 17]byte +var z2829 [1 << 17]byte +var z2830 [1 << 17]byte +var z2831 [1 << 17]byte +var z2832 [1 << 17]byte +var z2833 [1 << 17]byte +var z2834 [1 << 17]byte +var z2835 [1 << 17]byte +var z2836 [1 << 17]byte +var z2837 [1 << 17]byte +var z2838 [1 << 17]byte +var z2839 [1 << 17]byte +var z2840 [1 << 17]byte +var z2841 [1 << 17]byte +var z2842 [1 << 17]byte +var z2843 [1 << 17]byte +var z2844 [1 << 17]byte +var z2845 [1 << 17]byte +var z2846 [1 << 17]byte +var z2847 [1 << 17]byte +var z2848 [1 << 17]byte +var z2849 [1 << 17]byte +var z2850 [1 << 17]byte +var z2851 [1 << 17]byte +var z2852 [1 << 17]byte +var z2853 [1 << 17]byte +var z2854 [1 << 17]byte +var z2855 [1 << 17]byte +var z2856 [1 << 17]byte +var z2857 [1 << 17]byte +var z2858 [1 << 17]byte +var z2859 [1 << 17]byte +var z2860 [1 << 17]byte +var z2861 [1 << 17]byte +var z2862 [1 << 17]byte +var z2863 [1 << 17]byte +var z2864 [1 << 17]byte +var z2865 [1 << 17]byte +var z2866 [1 << 17]byte +var z2867 [1 << 17]byte +var z2868 [1 << 17]byte +var z2869 [1 << 17]byte +var z2870 [1 << 17]byte +var z2871 [1 << 17]byte +var z2872 [1 << 17]byte +var z2873 [1 << 17]byte +var z2874 [1 << 17]byte +var z2875 [1 << 17]byte +var z2876 [1 << 17]byte +var z2877 [1 << 17]byte +var z2878 [1 << 17]byte +var z2879 [1 << 17]byte +var z2880 [1 << 17]byte +var z2881 [1 << 17]byte +var z2882 [1 << 17]byte +var z2883 [1 << 17]byte +var z2884 [1 << 17]byte +var z2885 [1 << 17]byte +var z2886 [1 << 17]byte +var z2887 [1 << 17]byte +var z2888 [1 << 17]byte +var z2889 [1 << 17]byte +var z2890 [1 << 17]byte +var z2891 [1 << 17]byte +var z2892 [1 << 17]byte +var z2893 [1 << 17]byte +var z2894 [1 << 17]byte +var z2895 [1 << 17]byte +var z2896 [1 << 17]byte +var z2897 [1 << 17]byte +var z2898 [1 << 17]byte +var z2899 [1 << 17]byte +var z2900 [1 << 17]byte +var z2901 [1 << 17]byte +var z2902 [1 << 17]byte +var z2903 [1 << 17]byte +var z2904 [1 << 17]byte +var z2905 [1 << 17]byte +var z2906 [1 << 17]byte +var z2907 [1 << 17]byte +var z2908 [1 << 17]byte +var z2909 [1 << 17]byte +var z2910 [1 << 17]byte +var z2911 [1 << 17]byte +var z2912 [1 << 17]byte +var z2913 [1 << 17]byte +var z2914 [1 << 17]byte +var z2915 [1 << 17]byte +var z2916 [1 << 17]byte +var z2917 [1 << 17]byte +var z2918 [1 << 17]byte +var z2919 [1 << 17]byte +var z2920 [1 << 17]byte +var z2921 [1 << 17]byte +var z2922 [1 << 17]byte +var z2923 [1 << 17]byte +var z2924 [1 << 17]byte +var z2925 [1 << 17]byte +var z2926 [1 << 17]byte +var z2927 [1 << 17]byte +var z2928 [1 << 17]byte +var z2929 [1 << 17]byte +var z2930 [1 << 17]byte +var z2931 [1 << 17]byte +var z2932 [1 << 17]byte +var z2933 [1 << 17]byte +var z2934 [1 << 17]byte +var z2935 [1 << 17]byte +var z2936 [1 << 17]byte +var z2937 [1 << 17]byte +var z2938 [1 << 17]byte +var z2939 [1 << 17]byte +var z2940 [1 << 17]byte +var z2941 [1 << 17]byte +var z2942 [1 << 17]byte +var z2943 [1 << 17]byte +var z2944 [1 << 17]byte +var z2945 [1 << 17]byte +var z2946 [1 << 17]byte +var z2947 [1 << 17]byte +var z2948 [1 << 17]byte +var z2949 [1 << 17]byte +var z2950 [1 << 17]byte +var z2951 [1 << 17]byte +var z2952 [1 << 17]byte +var z2953 [1 << 17]byte +var z2954 [1 << 17]byte +var z2955 [1 << 17]byte +var z2956 [1 << 17]byte +var z2957 [1 << 17]byte +var z2958 [1 << 17]byte +var z2959 [1 << 17]byte +var z2960 [1 << 17]byte +var z2961 [1 << 17]byte +var z2962 [1 << 17]byte +var z2963 [1 << 17]byte +var z2964 [1 << 17]byte +var z2965 [1 << 17]byte +var z2966 [1 << 17]byte +var z2967 [1 << 17]byte +var z2968 [1 << 17]byte +var z2969 [1 << 17]byte +var z2970 [1 << 17]byte +var z2971 [1 << 17]byte +var z2972 [1 << 17]byte +var z2973 [1 << 17]byte +var z2974 [1 << 17]byte +var z2975 [1 << 17]byte +var z2976 [1 << 17]byte +var z2977 [1 << 17]byte +var z2978 [1 << 17]byte +var z2979 [1 << 17]byte +var z2980 [1 << 17]byte +var z2981 [1 << 17]byte +var z2982 [1 << 17]byte +var z2983 [1 << 17]byte +var z2984 [1 << 17]byte +var z2985 [1 << 17]byte +var z2986 [1 << 17]byte +var z2987 [1 << 17]byte +var z2988 [1 << 17]byte +var z2989 [1 << 17]byte +var z2990 [1 << 17]byte +var z2991 [1 << 17]byte +var z2992 [1 << 17]byte +var z2993 [1 << 17]byte +var z2994 [1 << 17]byte +var z2995 [1 << 17]byte +var z2996 [1 << 17]byte +var z2997 [1 << 17]byte +var z2998 [1 << 17]byte +var z2999 [1 << 17]byte +var z3000 [1 << 17]byte +var z3001 [1 << 17]byte +var z3002 [1 << 17]byte +var z3003 [1 << 17]byte +var z3004 [1 << 17]byte +var z3005 [1 << 17]byte +var z3006 [1 << 17]byte +var z3007 [1 << 17]byte +var z3008 [1 << 17]byte +var z3009 [1 << 17]byte +var z3010 [1 << 17]byte +var z3011 [1 << 17]byte +var z3012 [1 << 17]byte +var z3013 [1 << 17]byte +var z3014 [1 << 17]byte +var z3015 [1 << 17]byte +var z3016 [1 << 17]byte +var z3017 [1 << 17]byte +var z3018 [1 << 17]byte +var z3019 [1 << 17]byte +var z3020 [1 << 17]byte +var z3021 [1 << 17]byte +var z3022 [1 << 17]byte +var z3023 [1 << 17]byte +var z3024 [1 << 17]byte +var z3025 [1 << 17]byte +var z3026 [1 << 17]byte +var z3027 [1 << 17]byte +var z3028 [1 << 17]byte +var z3029 [1 << 17]byte +var z3030 [1 << 17]byte +var z3031 [1 << 17]byte +var z3032 [1 << 17]byte +var z3033 [1 << 17]byte +var z3034 [1 << 17]byte +var z3035 [1 << 17]byte +var z3036 [1 << 17]byte +var z3037 [1 << 17]byte +var z3038 [1 << 17]byte +var z3039 [1 << 17]byte +var z3040 [1 << 17]byte +var z3041 [1 << 17]byte +var z3042 [1 << 17]byte +var z3043 [1 << 17]byte +var z3044 [1 << 17]byte +var z3045 [1 << 17]byte +var z3046 [1 << 17]byte +var z3047 [1 << 17]byte +var z3048 [1 << 17]byte +var z3049 [1 << 17]byte +var z3050 [1 << 17]byte +var z3051 [1 << 17]byte +var z3052 [1 << 17]byte +var z3053 [1 << 17]byte +var z3054 [1 << 17]byte +var z3055 [1 << 17]byte +var z3056 [1 << 17]byte +var z3057 [1 << 17]byte +var z3058 [1 << 17]byte +var z3059 [1 << 17]byte +var z3060 [1 << 17]byte +var z3061 [1 << 17]byte +var z3062 [1 << 17]byte +var z3063 [1 << 17]byte +var z3064 [1 << 17]byte +var z3065 [1 << 17]byte +var z3066 [1 << 17]byte +var z3067 [1 << 17]byte +var z3068 [1 << 17]byte +var z3069 [1 << 17]byte +var z3070 [1 << 17]byte +var z3071 [1 << 17]byte +var z3072 [1 << 17]byte +var z3073 [1 << 17]byte +var z3074 [1 << 17]byte +var z3075 [1 << 17]byte +var z3076 [1 << 17]byte +var z3077 [1 << 17]byte +var z3078 [1 << 17]byte +var z3079 [1 << 17]byte +var z3080 [1 << 17]byte +var z3081 [1 << 17]byte +var z3082 [1 << 17]byte +var z3083 [1 << 17]byte +var z3084 [1 << 17]byte +var z3085 [1 << 17]byte +var z3086 [1 << 17]byte +var z3087 [1 << 17]byte +var z3088 [1 << 17]byte +var z3089 [1 << 17]byte +var z3090 [1 << 17]byte +var z3091 [1 << 17]byte +var z3092 [1 << 17]byte +var z3093 [1 << 17]byte +var z3094 [1 << 17]byte +var z3095 [1 << 17]byte +var z3096 [1 << 17]byte +var z3097 [1 << 17]byte +var z3098 [1 << 17]byte +var z3099 [1 << 17]byte +var z3100 [1 << 17]byte +var z3101 [1 << 17]byte +var z3102 [1 << 17]byte +var z3103 [1 << 17]byte +var z3104 [1 << 17]byte +var z3105 [1 << 17]byte +var z3106 [1 << 17]byte +var z3107 [1 << 17]byte +var z3108 [1 << 17]byte +var z3109 [1 << 17]byte +var z3110 [1 << 17]byte +var z3111 [1 << 17]byte +var z3112 [1 << 17]byte +var z3113 [1 << 17]byte +var z3114 [1 << 17]byte +var z3115 [1 << 17]byte +var z3116 [1 << 17]byte +var z3117 [1 << 17]byte +var z3118 [1 << 17]byte +var z3119 [1 << 17]byte +var z3120 [1 << 17]byte +var z3121 [1 << 17]byte +var z3122 [1 << 17]byte +var z3123 [1 << 17]byte +var z3124 [1 << 17]byte +var z3125 [1 << 17]byte +var z3126 [1 << 17]byte +var z3127 [1 << 17]byte +var z3128 [1 << 17]byte +var z3129 [1 << 17]byte +var z3130 [1 << 17]byte +var z3131 [1 << 17]byte +var z3132 [1 << 17]byte +var z3133 [1 << 17]byte +var z3134 [1 << 17]byte +var z3135 [1 << 17]byte +var z3136 [1 << 17]byte +var z3137 [1 << 17]byte +var z3138 [1 << 17]byte +var z3139 [1 << 17]byte +var z3140 [1 << 17]byte +var z3141 [1 << 17]byte +var z3142 [1 << 17]byte +var z3143 [1 << 17]byte +var z3144 [1 << 17]byte +var z3145 [1 << 17]byte +var z3146 [1 << 17]byte +var z3147 [1 << 17]byte +var z3148 [1 << 17]byte +var z3149 [1 << 17]byte +var z3150 [1 << 17]byte +var z3151 [1 << 17]byte +var z3152 [1 << 17]byte +var z3153 [1 << 17]byte +var z3154 [1 << 17]byte +var z3155 [1 << 17]byte +var z3156 [1 << 17]byte +var z3157 [1 << 17]byte +var z3158 [1 << 17]byte +var z3159 [1 << 17]byte +var z3160 [1 << 17]byte +var z3161 [1 << 17]byte +var z3162 [1 << 17]byte +var z3163 [1 << 17]byte +var z3164 [1 << 17]byte +var z3165 [1 << 17]byte +var z3166 [1 << 17]byte +var z3167 [1 << 17]byte +var z3168 [1 << 17]byte +var z3169 [1 << 17]byte +var z3170 [1 << 17]byte +var z3171 [1 << 17]byte +var z3172 [1 << 17]byte +var z3173 [1 << 17]byte +var z3174 [1 << 17]byte +var z3175 [1 << 17]byte +var z3176 [1 << 17]byte +var z3177 [1 << 17]byte +var z3178 [1 << 17]byte +var z3179 [1 << 17]byte +var z3180 [1 << 17]byte +var z3181 [1 << 17]byte +var z3182 [1 << 17]byte +var z3183 [1 << 17]byte +var z3184 [1 << 17]byte +var z3185 [1 << 17]byte +var z3186 [1 << 17]byte +var z3187 [1 << 17]byte +var z3188 [1 << 17]byte +var z3189 [1 << 17]byte +var z3190 [1 << 17]byte +var z3191 [1 << 17]byte +var z3192 [1 << 17]byte +var z3193 [1 << 17]byte +var z3194 [1 << 17]byte +var z3195 [1 << 17]byte +var z3196 [1 << 17]byte +var z3197 [1 << 17]byte +var z3198 [1 << 17]byte +var z3199 [1 << 17]byte +var z3200 [1 << 17]byte +var z3201 [1 << 17]byte +var z3202 [1 << 17]byte +var z3203 [1 << 17]byte +var z3204 [1 << 17]byte +var z3205 [1 << 17]byte +var z3206 [1 << 17]byte +var z3207 [1 << 17]byte +var z3208 [1 << 17]byte +var z3209 [1 << 17]byte +var z3210 [1 << 17]byte +var z3211 [1 << 17]byte +var z3212 [1 << 17]byte +var z3213 [1 << 17]byte +var z3214 [1 << 17]byte +var z3215 [1 << 17]byte +var z3216 [1 << 17]byte +var z3217 [1 << 17]byte +var z3218 [1 << 17]byte +var z3219 [1 << 17]byte +var z3220 [1 << 17]byte +var z3221 [1 << 17]byte +var z3222 [1 << 17]byte +var z3223 [1 << 17]byte +var z3224 [1 << 17]byte +var z3225 [1 << 17]byte +var z3226 [1 << 17]byte +var z3227 [1 << 17]byte +var z3228 [1 << 17]byte +var z3229 [1 << 17]byte +var z3230 [1 << 17]byte +var z3231 [1 << 17]byte +var z3232 [1 << 17]byte +var z3233 [1 << 17]byte +var z3234 [1 << 17]byte +var z3235 [1 << 17]byte +var z3236 [1 << 17]byte +var z3237 [1 << 17]byte +var z3238 [1 << 17]byte +var z3239 [1 << 17]byte +var z3240 [1 << 17]byte +var z3241 [1 << 17]byte +var z3242 [1 << 17]byte +var z3243 [1 << 17]byte +var z3244 [1 << 17]byte +var z3245 [1 << 17]byte +var z3246 [1 << 17]byte +var z3247 [1 << 17]byte +var z3248 [1 << 17]byte +var z3249 [1 << 17]byte +var z3250 [1 << 17]byte +var z3251 [1 << 17]byte +var z3252 [1 << 17]byte +var z3253 [1 << 17]byte +var z3254 [1 << 17]byte +var z3255 [1 << 17]byte +var z3256 [1 << 17]byte +var z3257 [1 << 17]byte +var z3258 [1 << 17]byte +var z3259 [1 << 17]byte +var z3260 [1 << 17]byte +var z3261 [1 << 17]byte +var z3262 [1 << 17]byte +var z3263 [1 << 17]byte +var z3264 [1 << 17]byte +var z3265 [1 << 17]byte +var z3266 [1 << 17]byte +var z3267 [1 << 17]byte +var z3268 [1 << 17]byte +var z3269 [1 << 17]byte +var z3270 [1 << 17]byte +var z3271 [1 << 17]byte +var z3272 [1 << 17]byte +var z3273 [1 << 17]byte +var z3274 [1 << 17]byte +var z3275 [1 << 17]byte +var z3276 [1 << 17]byte +var z3277 [1 << 17]byte +var z3278 [1 << 17]byte +var z3279 [1 << 17]byte +var z3280 [1 << 17]byte +var z3281 [1 << 17]byte +var z3282 [1 << 17]byte +var z3283 [1 << 17]byte +var z3284 [1 << 17]byte +var z3285 [1 << 17]byte +var z3286 [1 << 17]byte +var z3287 [1 << 17]byte +var z3288 [1 << 17]byte +var z3289 [1 << 17]byte +var z3290 [1 << 17]byte +var z3291 [1 << 17]byte +var z3292 [1 << 17]byte +var z3293 [1 << 17]byte +var z3294 [1 << 17]byte +var z3295 [1 << 17]byte +var z3296 [1 << 17]byte +var z3297 [1 << 17]byte +var z3298 [1 << 17]byte +var z3299 [1 << 17]byte +var z3300 [1 << 17]byte +var z3301 [1 << 17]byte +var z3302 [1 << 17]byte +var z3303 [1 << 17]byte +var z3304 [1 << 17]byte +var z3305 [1 << 17]byte +var z3306 [1 << 17]byte +var z3307 [1 << 17]byte +var z3308 [1 << 17]byte +var z3309 [1 << 17]byte +var z3310 [1 << 17]byte +var z3311 [1 << 17]byte +var z3312 [1 << 17]byte +var z3313 [1 << 17]byte +var z3314 [1 << 17]byte +var z3315 [1 << 17]byte +var z3316 [1 << 17]byte +var z3317 [1 << 17]byte +var z3318 [1 << 17]byte +var z3319 [1 << 17]byte +var z3320 [1 << 17]byte +var z3321 [1 << 17]byte +var z3322 [1 << 17]byte +var z3323 [1 << 17]byte +var z3324 [1 << 17]byte +var z3325 [1 << 17]byte +var z3326 [1 << 17]byte +var z3327 [1 << 17]byte +var z3328 [1 << 17]byte +var z3329 [1 << 17]byte +var z3330 [1 << 17]byte +var z3331 [1 << 17]byte +var z3332 [1 << 17]byte +var z3333 [1 << 17]byte +var z3334 [1 << 17]byte +var z3335 [1 << 17]byte +var z3336 [1 << 17]byte +var z3337 [1 << 17]byte +var z3338 [1 << 17]byte +var z3339 [1 << 17]byte +var z3340 [1 << 17]byte +var z3341 [1 << 17]byte +var z3342 [1 << 17]byte +var z3343 [1 << 17]byte +var z3344 [1 << 17]byte +var z3345 [1 << 17]byte +var z3346 [1 << 17]byte +var z3347 [1 << 17]byte +var z3348 [1 << 17]byte +var z3349 [1 << 17]byte +var z3350 [1 << 17]byte +var z3351 [1 << 17]byte +var z3352 [1 << 17]byte +var z3353 [1 << 17]byte +var z3354 [1 << 17]byte +var z3355 [1 << 17]byte +var z3356 [1 << 17]byte +var z3357 [1 << 17]byte +var z3358 [1 << 17]byte +var z3359 [1 << 17]byte +var z3360 [1 << 17]byte +var z3361 [1 << 17]byte +var z3362 [1 << 17]byte +var z3363 [1 << 17]byte +var z3364 [1 << 17]byte +var z3365 [1 << 17]byte +var z3366 [1 << 17]byte +var z3367 [1 << 17]byte +var z3368 [1 << 17]byte +var z3369 [1 << 17]byte +var z3370 [1 << 17]byte +var z3371 [1 << 17]byte +var z3372 [1 << 17]byte +var z3373 [1 << 17]byte +var z3374 [1 << 17]byte +var z3375 [1 << 17]byte +var z3376 [1 << 17]byte +var z3377 [1 << 17]byte +var z3378 [1 << 17]byte +var z3379 [1 << 17]byte +var z3380 [1 << 17]byte +var z3381 [1 << 17]byte +var z3382 [1 << 17]byte +var z3383 [1 << 17]byte +var z3384 [1 << 17]byte +var z3385 [1 << 17]byte +var z3386 [1 << 17]byte +var z3387 [1 << 17]byte +var z3388 [1 << 17]byte +var z3389 [1 << 17]byte +var z3390 [1 << 17]byte +var z3391 [1 << 17]byte +var z3392 [1 << 17]byte +var z3393 [1 << 17]byte +var z3394 [1 << 17]byte +var z3395 [1 << 17]byte +var z3396 [1 << 17]byte +var z3397 [1 << 17]byte +var z3398 [1 << 17]byte +var z3399 [1 << 17]byte +var z3400 [1 << 17]byte +var z3401 [1 << 17]byte +var z3402 [1 << 17]byte +var z3403 [1 << 17]byte +var z3404 [1 << 17]byte +var z3405 [1 << 17]byte +var z3406 [1 << 17]byte +var z3407 [1 << 17]byte +var z3408 [1 << 17]byte +var z3409 [1 << 17]byte +var z3410 [1 << 17]byte +var z3411 [1 << 17]byte +var z3412 [1 << 17]byte +var z3413 [1 << 17]byte +var z3414 [1 << 17]byte +var z3415 [1 << 17]byte +var z3416 [1 << 17]byte +var z3417 [1 << 17]byte +var z3418 [1 << 17]byte +var z3419 [1 << 17]byte +var z3420 [1 << 17]byte +var z3421 [1 << 17]byte +var z3422 [1 << 17]byte +var z3423 [1 << 17]byte +var z3424 [1 << 17]byte +var z3425 [1 << 17]byte +var z3426 [1 << 17]byte +var z3427 [1 << 17]byte +var z3428 [1 << 17]byte +var z3429 [1 << 17]byte +var z3430 [1 << 17]byte +var z3431 [1 << 17]byte +var z3432 [1 << 17]byte +var z3433 [1 << 17]byte +var z3434 [1 << 17]byte +var z3435 [1 << 17]byte +var z3436 [1 << 17]byte +var z3437 [1 << 17]byte +var z3438 [1 << 17]byte +var z3439 [1 << 17]byte +var z3440 [1 << 17]byte +var z3441 [1 << 17]byte +var z3442 [1 << 17]byte +var z3443 [1 << 17]byte +var z3444 [1 << 17]byte +var z3445 [1 << 17]byte +var z3446 [1 << 17]byte +var z3447 [1 << 17]byte +var z3448 [1 << 17]byte +var z3449 [1 << 17]byte +var z3450 [1 << 17]byte +var z3451 [1 << 17]byte +var z3452 [1 << 17]byte +var z3453 [1 << 17]byte +var z3454 [1 << 17]byte +var z3455 [1 << 17]byte +var z3456 [1 << 17]byte +var z3457 [1 << 17]byte +var z3458 [1 << 17]byte +var z3459 [1 << 17]byte +var z3460 [1 << 17]byte +var z3461 [1 << 17]byte +var z3462 [1 << 17]byte +var z3463 [1 << 17]byte +var z3464 [1 << 17]byte +var z3465 [1 << 17]byte +var z3466 [1 << 17]byte +var z3467 [1 << 17]byte +var z3468 [1 << 17]byte +var z3469 [1 << 17]byte +var z3470 [1 << 17]byte +var z3471 [1 << 17]byte +var z3472 [1 << 17]byte +var z3473 [1 << 17]byte +var z3474 [1 << 17]byte +var z3475 [1 << 17]byte +var z3476 [1 << 17]byte +var z3477 [1 << 17]byte +var z3478 [1 << 17]byte +var z3479 [1 << 17]byte +var z3480 [1 << 17]byte +var z3481 [1 << 17]byte +var z3482 [1 << 17]byte +var z3483 [1 << 17]byte +var z3484 [1 << 17]byte +var z3485 [1 << 17]byte +var z3486 [1 << 17]byte +var z3487 [1 << 17]byte +var z3488 [1 << 17]byte +var z3489 [1 << 17]byte +var z3490 [1 << 17]byte +var z3491 [1 << 17]byte +var z3492 [1 << 17]byte +var z3493 [1 << 17]byte +var z3494 [1 << 17]byte +var z3495 [1 << 17]byte +var z3496 [1 << 17]byte +var z3497 [1 << 17]byte +var z3498 [1 << 17]byte +var z3499 [1 << 17]byte +var z3500 [1 << 17]byte +var z3501 [1 << 17]byte +var z3502 [1 << 17]byte +var z3503 [1 << 17]byte +var z3504 [1 << 17]byte +var z3505 [1 << 17]byte +var z3506 [1 << 17]byte +var z3507 [1 << 17]byte +var z3508 [1 << 17]byte +var z3509 [1 << 17]byte +var z3510 [1 << 17]byte +var z3511 [1 << 17]byte +var z3512 [1 << 17]byte +var z3513 [1 << 17]byte +var z3514 [1 << 17]byte +var z3515 [1 << 17]byte +var z3516 [1 << 17]byte +var z3517 [1 << 17]byte +var z3518 [1 << 17]byte +var z3519 [1 << 17]byte +var z3520 [1 << 17]byte +var z3521 [1 << 17]byte +var z3522 [1 << 17]byte +var z3523 [1 << 17]byte +var z3524 [1 << 17]byte +var z3525 [1 << 17]byte +var z3526 [1 << 17]byte +var z3527 [1 << 17]byte +var z3528 [1 << 17]byte +var z3529 [1 << 17]byte +var z3530 [1 << 17]byte +var z3531 [1 << 17]byte +var z3532 [1 << 17]byte +var z3533 [1 << 17]byte +var z3534 [1 << 17]byte +var z3535 [1 << 17]byte +var z3536 [1 << 17]byte +var z3537 [1 << 17]byte +var z3538 [1 << 17]byte +var z3539 [1 << 17]byte +var z3540 [1 << 17]byte +var z3541 [1 << 17]byte +var z3542 [1 << 17]byte +var z3543 [1 << 17]byte +var z3544 [1 << 17]byte +var z3545 [1 << 17]byte +var z3546 [1 << 17]byte +var z3547 [1 << 17]byte +var z3548 [1 << 17]byte +var z3549 [1 << 17]byte +var z3550 [1 << 17]byte +var z3551 [1 << 17]byte +var z3552 [1 << 17]byte +var z3553 [1 << 17]byte +var z3554 [1 << 17]byte +var z3555 [1 << 17]byte +var z3556 [1 << 17]byte +var z3557 [1 << 17]byte +var z3558 [1 << 17]byte +var z3559 [1 << 17]byte +var z3560 [1 << 17]byte +var z3561 [1 << 17]byte +var z3562 [1 << 17]byte +var z3563 [1 << 17]byte +var z3564 [1 << 17]byte +var z3565 [1 << 17]byte +var z3566 [1 << 17]byte +var z3567 [1 << 17]byte +var z3568 [1 << 17]byte +var z3569 [1 << 17]byte +var z3570 [1 << 17]byte +var z3571 [1 << 17]byte +var z3572 [1 << 17]byte +var z3573 [1 << 17]byte +var z3574 [1 << 17]byte +var z3575 [1 << 17]byte +var z3576 [1 << 17]byte +var z3577 [1 << 17]byte +var z3578 [1 << 17]byte +var z3579 [1 << 17]byte +var z3580 [1 << 17]byte +var z3581 [1 << 17]byte +var z3582 [1 << 17]byte +var z3583 [1 << 17]byte +var z3584 [1 << 17]byte +var z3585 [1 << 17]byte +var z3586 [1 << 17]byte +var z3587 [1 << 17]byte +var z3588 [1 << 17]byte +var z3589 [1 << 17]byte +var z3590 [1 << 17]byte +var z3591 [1 << 17]byte +var z3592 [1 << 17]byte +var z3593 [1 << 17]byte +var z3594 [1 << 17]byte +var z3595 [1 << 17]byte +var z3596 [1 << 17]byte +var z3597 [1 << 17]byte +var z3598 [1 << 17]byte +var z3599 [1 << 17]byte +var z3600 [1 << 17]byte +var z3601 [1 << 17]byte +var z3602 [1 << 17]byte +var z3603 [1 << 17]byte +var z3604 [1 << 17]byte +var z3605 [1 << 17]byte +var z3606 [1 << 17]byte +var z3607 [1 << 17]byte +var z3608 [1 << 17]byte +var z3609 [1 << 17]byte +var z3610 [1 << 17]byte +var z3611 [1 << 17]byte +var z3612 [1 << 17]byte +var z3613 [1 << 17]byte +var z3614 [1 << 17]byte +var z3615 [1 << 17]byte +var z3616 [1 << 17]byte +var z3617 [1 << 17]byte +var z3618 [1 << 17]byte +var z3619 [1 << 17]byte +var z3620 [1 << 17]byte +var z3621 [1 << 17]byte +var z3622 [1 << 17]byte +var z3623 [1 << 17]byte +var z3624 [1 << 17]byte +var z3625 [1 << 17]byte +var z3626 [1 << 17]byte +var z3627 [1 << 17]byte +var z3628 [1 << 17]byte +var z3629 [1 << 17]byte +var z3630 [1 << 17]byte +var z3631 [1 << 17]byte +var z3632 [1 << 17]byte +var z3633 [1 << 17]byte +var z3634 [1 << 17]byte +var z3635 [1 << 17]byte +var z3636 [1 << 17]byte +var z3637 [1 << 17]byte +var z3638 [1 << 17]byte +var z3639 [1 << 17]byte +var z3640 [1 << 17]byte +var z3641 [1 << 17]byte +var z3642 [1 << 17]byte +var z3643 [1 << 17]byte +var z3644 [1 << 17]byte +var z3645 [1 << 17]byte +var z3646 [1 << 17]byte +var z3647 [1 << 17]byte +var z3648 [1 << 17]byte +var z3649 [1 << 17]byte +var z3650 [1 << 17]byte +var z3651 [1 << 17]byte +var z3652 [1 << 17]byte +var z3653 [1 << 17]byte +var z3654 [1 << 17]byte +var z3655 [1 << 17]byte +var z3656 [1 << 17]byte +var z3657 [1 << 17]byte +var z3658 [1 << 17]byte +var z3659 [1 << 17]byte +var z3660 [1 << 17]byte +var z3661 [1 << 17]byte +var z3662 [1 << 17]byte +var z3663 [1 << 17]byte +var z3664 [1 << 17]byte +var z3665 [1 << 17]byte +var z3666 [1 << 17]byte +var z3667 [1 << 17]byte +var z3668 [1 << 17]byte +var z3669 [1 << 17]byte +var z3670 [1 << 17]byte +var z3671 [1 << 17]byte +var z3672 [1 << 17]byte +var z3673 [1 << 17]byte +var z3674 [1 << 17]byte +var z3675 [1 << 17]byte +var z3676 [1 << 17]byte +var z3677 [1 << 17]byte +var z3678 [1 << 17]byte +var z3679 [1 << 17]byte +var z3680 [1 << 17]byte +var z3681 [1 << 17]byte +var z3682 [1 << 17]byte +var z3683 [1 << 17]byte +var z3684 [1 << 17]byte +var z3685 [1 << 17]byte +var z3686 [1 << 17]byte +var z3687 [1 << 17]byte +var z3688 [1 << 17]byte +var z3689 [1 << 17]byte +var z3690 [1 << 17]byte +var z3691 [1 << 17]byte +var z3692 [1 << 17]byte +var z3693 [1 << 17]byte +var z3694 [1 << 17]byte +var z3695 [1 << 17]byte +var z3696 [1 << 17]byte +var z3697 [1 << 17]byte +var z3698 [1 << 17]byte +var z3699 [1 << 17]byte +var z3700 [1 << 17]byte +var z3701 [1 << 17]byte +var z3702 [1 << 17]byte +var z3703 [1 << 17]byte +var z3704 [1 << 17]byte +var z3705 [1 << 17]byte +var z3706 [1 << 17]byte +var z3707 [1 << 17]byte +var z3708 [1 << 17]byte +var z3709 [1 << 17]byte +var z3710 [1 << 17]byte +var z3711 [1 << 17]byte +var z3712 [1 << 17]byte +var z3713 [1 << 17]byte +var z3714 [1 << 17]byte +var z3715 [1 << 17]byte +var z3716 [1 << 17]byte +var z3717 [1 << 17]byte +var z3718 [1 << 17]byte +var z3719 [1 << 17]byte +var z3720 [1 << 17]byte +var z3721 [1 << 17]byte +var z3722 [1 << 17]byte +var z3723 [1 << 17]byte +var z3724 [1 << 17]byte +var z3725 [1 << 17]byte +var z3726 [1 << 17]byte +var z3727 [1 << 17]byte +var z3728 [1 << 17]byte +var z3729 [1 << 17]byte +var z3730 [1 << 17]byte +var z3731 [1 << 17]byte +var z3732 [1 << 17]byte +var z3733 [1 << 17]byte +var z3734 [1 << 17]byte +var z3735 [1 << 17]byte +var z3736 [1 << 17]byte +var z3737 [1 << 17]byte +var z3738 [1 << 17]byte +var z3739 [1 << 17]byte +var z3740 [1 << 17]byte +var z3741 [1 << 17]byte +var z3742 [1 << 17]byte +var z3743 [1 << 17]byte +var z3744 [1 << 17]byte +var z3745 [1 << 17]byte +var z3746 [1 << 17]byte +var z3747 [1 << 17]byte +var z3748 [1 << 17]byte +var z3749 [1 << 17]byte +var z3750 [1 << 17]byte +var z3751 [1 << 17]byte +var z3752 [1 << 17]byte +var z3753 [1 << 17]byte +var z3754 [1 << 17]byte +var z3755 [1 << 17]byte +var z3756 [1 << 17]byte +var z3757 [1 << 17]byte +var z3758 [1 << 17]byte +var z3759 [1 << 17]byte +var z3760 [1 << 17]byte +var z3761 [1 << 17]byte +var z3762 [1 << 17]byte +var z3763 [1 << 17]byte +var z3764 [1 << 17]byte +var z3765 [1 << 17]byte +var z3766 [1 << 17]byte +var z3767 [1 << 17]byte +var z3768 [1 << 17]byte +var z3769 [1 << 17]byte +var z3770 [1 << 17]byte +var z3771 [1 << 17]byte +var z3772 [1 << 17]byte +var z3773 [1 << 17]byte +var z3774 [1 << 17]byte +var z3775 [1 << 17]byte +var z3776 [1 << 17]byte +var z3777 [1 << 17]byte +var z3778 [1 << 17]byte +var z3779 [1 << 17]byte +var z3780 [1 << 17]byte +var z3781 [1 << 17]byte +var z3782 [1 << 17]byte +var z3783 [1 << 17]byte +var z3784 [1 << 17]byte +var z3785 [1 << 17]byte +var z3786 [1 << 17]byte +var z3787 [1 << 17]byte +var z3788 [1 << 17]byte +var z3789 [1 << 17]byte +var z3790 [1 << 17]byte +var z3791 [1 << 17]byte +var z3792 [1 << 17]byte +var z3793 [1 << 17]byte +var z3794 [1 << 17]byte +var z3795 [1 << 17]byte +var z3796 [1 << 17]byte +var z3797 [1 << 17]byte +var z3798 [1 << 17]byte +var z3799 [1 << 17]byte +var z3800 [1 << 17]byte +var z3801 [1 << 17]byte +var z3802 [1 << 17]byte +var z3803 [1 << 17]byte +var z3804 [1 << 17]byte +var z3805 [1 << 17]byte +var z3806 [1 << 17]byte +var z3807 [1 << 17]byte +var z3808 [1 << 17]byte +var z3809 [1 << 17]byte +var z3810 [1 << 17]byte +var z3811 [1 << 17]byte +var z3812 [1 << 17]byte +var z3813 [1 << 17]byte +var z3814 [1 << 17]byte +var z3815 [1 << 17]byte +var z3816 [1 << 17]byte +var z3817 [1 << 17]byte +var z3818 [1 << 17]byte +var z3819 [1 << 17]byte +var z3820 [1 << 17]byte +var z3821 [1 << 17]byte +var z3822 [1 << 17]byte +var z3823 [1 << 17]byte +var z3824 [1 << 17]byte +var z3825 [1 << 17]byte +var z3826 [1 << 17]byte +var z3827 [1 << 17]byte +var z3828 [1 << 17]byte +var z3829 [1 << 17]byte +var z3830 [1 << 17]byte +var z3831 [1 << 17]byte +var z3832 [1 << 17]byte +var z3833 [1 << 17]byte +var z3834 [1 << 17]byte +var z3835 [1 << 17]byte +var z3836 [1 << 17]byte +var z3837 [1 << 17]byte +var z3838 [1 << 17]byte +var z3839 [1 << 17]byte +var z3840 [1 << 17]byte +var z3841 [1 << 17]byte +var z3842 [1 << 17]byte +var z3843 [1 << 17]byte +var z3844 [1 << 17]byte +var z3845 [1 << 17]byte +var z3846 [1 << 17]byte +var z3847 [1 << 17]byte +var z3848 [1 << 17]byte +var z3849 [1 << 17]byte +var z3850 [1 << 17]byte +var z3851 [1 << 17]byte +var z3852 [1 << 17]byte +var z3853 [1 << 17]byte +var z3854 [1 << 17]byte +var z3855 [1 << 17]byte +var z3856 [1 << 17]byte +var z3857 [1 << 17]byte +var z3858 [1 << 17]byte +var z3859 [1 << 17]byte +var z3860 [1 << 17]byte +var z3861 [1 << 17]byte +var z3862 [1 << 17]byte +var z3863 [1 << 17]byte +var z3864 [1 << 17]byte +var z3865 [1 << 17]byte +var z3866 [1 << 17]byte +var z3867 [1 << 17]byte +var z3868 [1 << 17]byte +var z3869 [1 << 17]byte +var z3870 [1 << 17]byte +var z3871 [1 << 17]byte +var z3872 [1 << 17]byte +var z3873 [1 << 17]byte +var z3874 [1 << 17]byte +var z3875 [1 << 17]byte +var z3876 [1 << 17]byte +var z3877 [1 << 17]byte +var z3878 [1 << 17]byte +var z3879 [1 << 17]byte +var z3880 [1 << 17]byte +var z3881 [1 << 17]byte +var z3882 [1 << 17]byte +var z3883 [1 << 17]byte +var z3884 [1 << 17]byte +var z3885 [1 << 17]byte +var z3886 [1 << 17]byte +var z3887 [1 << 17]byte +var z3888 [1 << 17]byte +var z3889 [1 << 17]byte +var z3890 [1 << 17]byte +var z3891 [1 << 17]byte +var z3892 [1 << 17]byte +var z3893 [1 << 17]byte +var z3894 [1 << 17]byte +var z3895 [1 << 17]byte +var z3896 [1 << 17]byte +var z3897 [1 << 17]byte +var z3898 [1 << 17]byte +var z3899 [1 << 17]byte +var z3900 [1 << 17]byte +var z3901 [1 << 17]byte +var z3902 [1 << 17]byte +var z3903 [1 << 17]byte +var z3904 [1 << 17]byte +var z3905 [1 << 17]byte +var z3906 [1 << 17]byte +var z3907 [1 << 17]byte +var z3908 [1 << 17]byte +var z3909 [1 << 17]byte +var z3910 [1 << 17]byte +var z3911 [1 << 17]byte +var z3912 [1 << 17]byte +var z3913 [1 << 17]byte +var z3914 [1 << 17]byte +var z3915 [1 << 17]byte +var z3916 [1 << 17]byte +var z3917 [1 << 17]byte +var z3918 [1 << 17]byte +var z3919 [1 << 17]byte +var z3920 [1 << 17]byte +var z3921 [1 << 17]byte +var z3922 [1 << 17]byte +var z3923 [1 << 17]byte +var z3924 [1 << 17]byte +var z3925 [1 << 17]byte +var z3926 [1 << 17]byte +var z3927 [1 << 17]byte +var z3928 [1 << 17]byte +var z3929 [1 << 17]byte +var z3930 [1 << 17]byte +var z3931 [1 << 17]byte +var z3932 [1 << 17]byte +var z3933 [1 << 17]byte +var z3934 [1 << 17]byte +var z3935 [1 << 17]byte +var z3936 [1 << 17]byte +var z3937 [1 << 17]byte +var z3938 [1 << 17]byte +var z3939 [1 << 17]byte +var z3940 [1 << 17]byte +var z3941 [1 << 17]byte +var z3942 [1 << 17]byte +var z3943 [1 << 17]byte +var z3944 [1 << 17]byte +var z3945 [1 << 17]byte +var z3946 [1 << 17]byte +var z3947 [1 << 17]byte +var z3948 [1 << 17]byte +var z3949 [1 << 17]byte +var z3950 [1 << 17]byte +var z3951 [1 << 17]byte +var z3952 [1 << 17]byte +var z3953 [1 << 17]byte +var z3954 [1 << 17]byte +var z3955 [1 << 17]byte +var z3956 [1 << 17]byte +var z3957 [1 << 17]byte +var z3958 [1 << 17]byte +var z3959 [1 << 17]byte +var z3960 [1 << 17]byte +var z3961 [1 << 17]byte +var z3962 [1 << 17]byte +var z3963 [1 << 17]byte +var z3964 [1 << 17]byte +var z3965 [1 << 17]byte +var z3966 [1 << 17]byte +var z3967 [1 << 17]byte +var z3968 [1 << 17]byte +var z3969 [1 << 17]byte +var z3970 [1 << 17]byte +var z3971 [1 << 17]byte +var z3972 [1 << 17]byte +var z3973 [1 << 17]byte +var z3974 [1 << 17]byte +var z3975 [1 << 17]byte +var z3976 [1 << 17]byte +var z3977 [1 << 17]byte +var z3978 [1 << 17]byte +var z3979 [1 << 17]byte +var z3980 [1 << 17]byte +var z3981 [1 << 17]byte +var z3982 [1 << 17]byte +var z3983 [1 << 17]byte +var z3984 [1 << 17]byte +var z3985 [1 << 17]byte +var z3986 [1 << 17]byte +var z3987 [1 << 17]byte +var z3988 [1 << 17]byte +var z3989 [1 << 17]byte +var z3990 [1 << 17]byte +var z3991 [1 << 17]byte +var z3992 [1 << 17]byte +var z3993 [1 << 17]byte +var z3994 [1 << 17]byte +var z3995 [1 << 17]byte +var z3996 [1 << 17]byte +var z3997 [1 << 17]byte +var z3998 [1 << 17]byte +var z3999 [1 << 17]byte +var z4000 [1 << 17]byte +var z4001 [1 << 17]byte +var z4002 [1 << 17]byte +var z4003 [1 << 17]byte +var z4004 [1 << 17]byte +var z4005 [1 << 17]byte +var z4006 [1 << 17]byte +var z4007 [1 << 17]byte +var z4008 [1 << 17]byte +var z4009 [1 << 17]byte +var z4010 [1 << 17]byte +var z4011 [1 << 17]byte +var z4012 [1 << 17]byte +var z4013 [1 << 17]byte +var z4014 [1 << 17]byte +var z4015 [1 << 17]byte +var z4016 [1 << 17]byte +var z4017 [1 << 17]byte +var z4018 [1 << 17]byte +var z4019 [1 << 17]byte +var z4020 [1 << 17]byte +var z4021 [1 << 17]byte +var z4022 [1 << 17]byte +var z4023 [1 << 17]byte +var z4024 [1 << 17]byte +var z4025 [1 << 17]byte +var z4026 [1 << 17]byte +var z4027 [1 << 17]byte +var z4028 [1 << 17]byte +var z4029 [1 << 17]byte +var z4030 [1 << 17]byte +var z4031 [1 << 17]byte +var z4032 [1 << 17]byte +var z4033 [1 << 17]byte +var z4034 [1 << 17]byte +var z4035 [1 << 17]byte +var z4036 [1 << 17]byte +var z4037 [1 << 17]byte +var z4038 [1 << 17]byte +var z4039 [1 << 17]byte +var z4040 [1 << 17]byte +var z4041 [1 << 17]byte +var z4042 [1 << 17]byte +var z4043 [1 << 17]byte +var z4044 [1 << 17]byte +var z4045 [1 << 17]byte +var z4046 [1 << 17]byte +var z4047 [1 << 17]byte +var z4048 [1 << 17]byte +var z4049 [1 << 17]byte +var z4050 [1 << 17]byte +var z4051 [1 << 17]byte +var z4052 [1 << 17]byte +var z4053 [1 << 17]byte +var z4054 [1 << 17]byte +var z4055 [1 << 17]byte +var z4056 [1 << 17]byte +var z4057 [1 << 17]byte +var z4058 [1 << 17]byte +var z4059 [1 << 17]byte +var z4060 [1 << 17]byte +var z4061 [1 << 17]byte +var z4062 [1 << 17]byte +var z4063 [1 << 17]byte +var z4064 [1 << 17]byte +var z4065 [1 << 17]byte +var z4066 [1 << 17]byte +var z4067 [1 << 17]byte +var z4068 [1 << 17]byte +var z4069 [1 << 17]byte +var z4070 [1 << 17]byte +var z4071 [1 << 17]byte +var z4072 [1 << 17]byte +var z4073 [1 << 17]byte +var z4074 [1 << 17]byte +var z4075 [1 << 17]byte +var z4076 [1 << 17]byte +var z4077 [1 << 17]byte +var z4078 [1 << 17]byte +var z4079 [1 << 17]byte +var z4080 [1 << 17]byte +var z4081 [1 << 17]byte +var z4082 [1 << 17]byte +var z4083 [1 << 17]byte +var z4084 [1 << 17]byte +var z4085 [1 << 17]byte +var z4086 [1 << 17]byte +var z4087 [1 << 17]byte +var z4088 [1 << 17]byte +var z4089 [1 << 17]byte +var z4090 [1 << 17]byte +var z4091 [1 << 17]byte +var z4092 [1 << 17]byte +var z4093 [1 << 17]byte +var z4094 [1 << 17]byte +var z4095 [1 << 17]byte +var z4096 [1 << 17]byte +var z4097 [1 << 17]byte +var z4098 [1 << 17]byte +var z4099 [1 << 17]byte +var z4100 [1 << 17]byte +var z4101 [1 << 17]byte +var z4102 [1 << 17]byte +var z4103 [1 << 17]byte +var z4104 [1 << 17]byte +var z4105 [1 << 17]byte +var z4106 [1 << 17]byte +var z4107 [1 << 17]byte +var z4108 [1 << 17]byte +var z4109 [1 << 17]byte +var z4110 [1 << 17]byte +var z4111 [1 << 17]byte +var z4112 [1 << 17]byte +var z4113 [1 << 17]byte +var z4114 [1 << 17]byte +var z4115 [1 << 17]byte +var z4116 [1 << 17]byte +var z4117 [1 << 17]byte +var z4118 [1 << 17]byte +var z4119 [1 << 17]byte +var z4120 [1 << 17]byte +var z4121 [1 << 17]byte +var z4122 [1 << 17]byte +var z4123 [1 << 17]byte +var z4124 [1 << 17]byte +var z4125 [1 << 17]byte +var z4126 [1 << 17]byte +var z4127 [1 << 17]byte +var z4128 [1 << 17]byte +var z4129 [1 << 17]byte +var z4130 [1 << 17]byte +var z4131 [1 << 17]byte +var z4132 [1 << 17]byte +var z4133 [1 << 17]byte +var z4134 [1 << 17]byte +var z4135 [1 << 17]byte +var z4136 [1 << 17]byte +var z4137 [1 << 17]byte +var z4138 [1 << 17]byte +var z4139 [1 << 17]byte +var z4140 [1 << 17]byte +var z4141 [1 << 17]byte +var z4142 [1 << 17]byte +var z4143 [1 << 17]byte +var z4144 [1 << 17]byte +var z4145 [1 << 17]byte +var z4146 [1 << 17]byte +var z4147 [1 << 17]byte +var z4148 [1 << 17]byte +var z4149 [1 << 17]byte +var z4150 [1 << 17]byte +var z4151 [1 << 17]byte +var z4152 [1 << 17]byte +var z4153 [1 << 17]byte +var z4154 [1 << 17]byte +var z4155 [1 << 17]byte +var z4156 [1 << 17]byte +var z4157 [1 << 17]byte +var z4158 [1 << 17]byte +var z4159 [1 << 17]byte +var z4160 [1 << 17]byte +var z4161 [1 << 17]byte +var z4162 [1 << 17]byte +var z4163 [1 << 17]byte +var z4164 [1 << 17]byte +var z4165 [1 << 17]byte +var z4166 [1 << 17]byte +var z4167 [1 << 17]byte +var z4168 [1 << 17]byte +var z4169 [1 << 17]byte +var z4170 [1 << 17]byte +var z4171 [1 << 17]byte +var z4172 [1 << 17]byte +var z4173 [1 << 17]byte +var z4174 [1 << 17]byte +var z4175 [1 << 17]byte +var z4176 [1 << 17]byte +var z4177 [1 << 17]byte +var z4178 [1 << 17]byte +var z4179 [1 << 17]byte +var z4180 [1 << 17]byte +var z4181 [1 << 17]byte +var z4182 [1 << 17]byte +var z4183 [1 << 17]byte +var z4184 [1 << 17]byte +var z4185 [1 << 17]byte +var z4186 [1 << 17]byte +var z4187 [1 << 17]byte +var z4188 [1 << 17]byte +var z4189 [1 << 17]byte +var z4190 [1 << 17]byte +var z4191 [1 << 17]byte +var z4192 [1 << 17]byte +var z4193 [1 << 17]byte +var z4194 [1 << 17]byte +var z4195 [1 << 17]byte +var z4196 [1 << 17]byte +var z4197 [1 << 17]byte +var z4198 [1 << 17]byte +var z4199 [1 << 17]byte +var z4200 [1 << 17]byte +var z4201 [1 << 17]byte +var z4202 [1 << 17]byte +var z4203 [1 << 17]byte +var z4204 [1 << 17]byte +var z4205 [1 << 17]byte +var z4206 [1 << 17]byte +var z4207 [1 << 17]byte +var z4208 [1 << 17]byte +var z4209 [1 << 17]byte +var z4210 [1 << 17]byte +var z4211 [1 << 17]byte +var z4212 [1 << 17]byte +var z4213 [1 << 17]byte +var z4214 [1 << 17]byte +var z4215 [1 << 17]byte +var z4216 [1 << 17]byte +var z4217 [1 << 17]byte +var z4218 [1 << 17]byte +var z4219 [1 << 17]byte +var z4220 [1 << 17]byte +var z4221 [1 << 17]byte +var z4222 [1 << 17]byte +var z4223 [1 << 17]byte +var z4224 [1 << 17]byte +var z4225 [1 << 17]byte +var z4226 [1 << 17]byte +var z4227 [1 << 17]byte +var z4228 [1 << 17]byte +var z4229 [1 << 17]byte +var z4230 [1 << 17]byte +var z4231 [1 << 17]byte +var z4232 [1 << 17]byte +var z4233 [1 << 17]byte +var z4234 [1 << 17]byte +var z4235 [1 << 17]byte +var z4236 [1 << 17]byte +var z4237 [1 << 17]byte +var z4238 [1 << 17]byte +var z4239 [1 << 17]byte +var z4240 [1 << 17]byte +var z4241 [1 << 17]byte +var z4242 [1 << 17]byte +var z4243 [1 << 17]byte +var z4244 [1 << 17]byte +var z4245 [1 << 17]byte +var z4246 [1 << 17]byte +var z4247 [1 << 17]byte +var z4248 [1 << 17]byte +var z4249 [1 << 17]byte +var z4250 [1 << 17]byte +var z4251 [1 << 17]byte +var z4252 [1 << 17]byte +var z4253 [1 << 17]byte +var z4254 [1 << 17]byte +var z4255 [1 << 17]byte +var z4256 [1 << 17]byte +var z4257 [1 << 17]byte +var z4258 [1 << 17]byte +var z4259 [1 << 17]byte +var z4260 [1 << 17]byte +var z4261 [1 << 17]byte +var z4262 [1 << 17]byte +var z4263 [1 << 17]byte +var z4264 [1 << 17]byte +var z4265 [1 << 17]byte +var z4266 [1 << 17]byte +var z4267 [1 << 17]byte +var z4268 [1 << 17]byte +var z4269 [1 << 17]byte +var z4270 [1 << 17]byte +var z4271 [1 << 17]byte +var z4272 [1 << 17]byte +var z4273 [1 << 17]byte +var z4274 [1 << 17]byte +var z4275 [1 << 17]byte +var z4276 [1 << 17]byte +var z4277 [1 << 17]byte +var z4278 [1 << 17]byte +var z4279 [1 << 17]byte +var z4280 [1 << 17]byte +var z4281 [1 << 17]byte +var z4282 [1 << 17]byte +var z4283 [1 << 17]byte +var z4284 [1 << 17]byte +var z4285 [1 << 17]byte +var z4286 [1 << 17]byte +var z4287 [1 << 17]byte +var z4288 [1 << 17]byte +var z4289 [1 << 17]byte +var z4290 [1 << 17]byte +var z4291 [1 << 17]byte +var z4292 [1 << 17]byte +var z4293 [1 << 17]byte +var z4294 [1 << 17]byte +var z4295 [1 << 17]byte +var z4296 [1 << 17]byte +var z4297 [1 << 17]byte +var z4298 [1 << 17]byte +var z4299 [1 << 17]byte +var z4300 [1 << 17]byte +var z4301 [1 << 17]byte +var z4302 [1 << 17]byte +var z4303 [1 << 17]byte +var z4304 [1 << 17]byte +var z4305 [1 << 17]byte +var z4306 [1 << 17]byte +var z4307 [1 << 17]byte +var z4308 [1 << 17]byte +var z4309 [1 << 17]byte +var z4310 [1 << 17]byte +var z4311 [1 << 17]byte +var z4312 [1 << 17]byte +var z4313 [1 << 17]byte +var z4314 [1 << 17]byte +var z4315 [1 << 17]byte +var z4316 [1 << 17]byte +var z4317 [1 << 17]byte +var z4318 [1 << 17]byte +var z4319 [1 << 17]byte +var z4320 [1 << 17]byte +var z4321 [1 << 17]byte +var z4322 [1 << 17]byte +var z4323 [1 << 17]byte +var z4324 [1 << 17]byte +var z4325 [1 << 17]byte +var z4326 [1 << 17]byte +var z4327 [1 << 17]byte +var z4328 [1 << 17]byte +var z4329 [1 << 17]byte +var z4330 [1 << 17]byte +var z4331 [1 << 17]byte +var z4332 [1 << 17]byte +var z4333 [1 << 17]byte +var z4334 [1 << 17]byte +var z4335 [1 << 17]byte +var z4336 [1 << 17]byte +var z4337 [1 << 17]byte +var z4338 [1 << 17]byte +var z4339 [1 << 17]byte +var z4340 [1 << 17]byte +var z4341 [1 << 17]byte +var z4342 [1 << 17]byte +var z4343 [1 << 17]byte +var z4344 [1 << 17]byte +var z4345 [1 << 17]byte +var z4346 [1 << 17]byte +var z4347 [1 << 17]byte +var z4348 [1 << 17]byte +var z4349 [1 << 17]byte +var z4350 [1 << 17]byte +var z4351 [1 << 17]byte +var z4352 [1 << 17]byte +var z4353 [1 << 17]byte +var z4354 [1 << 17]byte +var z4355 [1 << 17]byte +var z4356 [1 << 17]byte +var z4357 [1 << 17]byte +var z4358 [1 << 17]byte +var z4359 [1 << 17]byte +var z4360 [1 << 17]byte +var z4361 [1 << 17]byte +var z4362 [1 << 17]byte +var z4363 [1 << 17]byte +var z4364 [1 << 17]byte +var z4365 [1 << 17]byte +var z4366 [1 << 17]byte +var z4367 [1 << 17]byte +var z4368 [1 << 17]byte +var z4369 [1 << 17]byte +var z4370 [1 << 17]byte +var z4371 [1 << 17]byte +var z4372 [1 << 17]byte +var z4373 [1 << 17]byte +var z4374 [1 << 17]byte +var z4375 [1 << 17]byte +var z4376 [1 << 17]byte +var z4377 [1 << 17]byte +var z4378 [1 << 17]byte +var z4379 [1 << 17]byte +var z4380 [1 << 17]byte +var z4381 [1 << 17]byte +var z4382 [1 << 17]byte +var z4383 [1 << 17]byte +var z4384 [1 << 17]byte +var z4385 [1 << 17]byte +var z4386 [1 << 17]byte +var z4387 [1 << 17]byte +var z4388 [1 << 17]byte +var z4389 [1 << 17]byte +var z4390 [1 << 17]byte +var z4391 [1 << 17]byte +var z4392 [1 << 17]byte +var z4393 [1 << 17]byte +var z4394 [1 << 17]byte +var z4395 [1 << 17]byte +var z4396 [1 << 17]byte +var z4397 [1 << 17]byte +var z4398 [1 << 17]byte +var z4399 [1 << 17]byte +var z4400 [1 << 17]byte +var z4401 [1 << 17]byte +var z4402 [1 << 17]byte +var z4403 [1 << 17]byte +var z4404 [1 << 17]byte +var z4405 [1 << 17]byte +var z4406 [1 << 17]byte +var z4407 [1 << 17]byte +var z4408 [1 << 17]byte +var z4409 [1 << 17]byte +var z4410 [1 << 17]byte +var z4411 [1 << 17]byte +var z4412 [1 << 17]byte +var z4413 [1 << 17]byte +var z4414 [1 << 17]byte +var z4415 [1 << 17]byte +var z4416 [1 << 17]byte +var z4417 [1 << 17]byte +var z4418 [1 << 17]byte +var z4419 [1 << 17]byte +var z4420 [1 << 17]byte +var z4421 [1 << 17]byte +var z4422 [1 << 17]byte +var z4423 [1 << 17]byte +var z4424 [1 << 17]byte +var z4425 [1 << 17]byte +var z4426 [1 << 17]byte +var z4427 [1 << 17]byte +var z4428 [1 << 17]byte +var z4429 [1 << 17]byte +var z4430 [1 << 17]byte +var z4431 [1 << 17]byte +var z4432 [1 << 17]byte +var z4433 [1 << 17]byte +var z4434 [1 << 17]byte +var z4435 [1 << 17]byte +var z4436 [1 << 17]byte +var z4437 [1 << 17]byte +var z4438 [1 << 17]byte +var z4439 [1 << 17]byte +var z4440 [1 << 17]byte +var z4441 [1 << 17]byte +var z4442 [1 << 17]byte +var z4443 [1 << 17]byte +var z4444 [1 << 17]byte +var z4445 [1 << 17]byte +var z4446 [1 << 17]byte +var z4447 [1 << 17]byte +var z4448 [1 << 17]byte +var z4449 [1 << 17]byte +var z4450 [1 << 17]byte +var z4451 [1 << 17]byte +var z4452 [1 << 17]byte +var z4453 [1 << 17]byte +var z4454 [1 << 17]byte +var z4455 [1 << 17]byte +var z4456 [1 << 17]byte +var z4457 [1 << 17]byte +var z4458 [1 << 17]byte +var z4459 [1 << 17]byte +var z4460 [1 << 17]byte +var z4461 [1 << 17]byte +var z4462 [1 << 17]byte +var z4463 [1 << 17]byte +var z4464 [1 << 17]byte +var z4465 [1 << 17]byte +var z4466 [1 << 17]byte +var z4467 [1 << 17]byte +var z4468 [1 << 17]byte +var z4469 [1 << 17]byte +var z4470 [1 << 17]byte +var z4471 [1 << 17]byte +var z4472 [1 << 17]byte +var z4473 [1 << 17]byte +var z4474 [1 << 17]byte +var z4475 [1 << 17]byte +var z4476 [1 << 17]byte +var z4477 [1 << 17]byte +var z4478 [1 << 17]byte +var z4479 [1 << 17]byte +var z4480 [1 << 17]byte +var z4481 [1 << 17]byte +var z4482 [1 << 17]byte +var z4483 [1 << 17]byte +var z4484 [1 << 17]byte +var z4485 [1 << 17]byte +var z4486 [1 << 17]byte +var z4487 [1 << 17]byte +var z4488 [1 << 17]byte +var z4489 [1 << 17]byte +var z4490 [1 << 17]byte +var z4491 [1 << 17]byte +var z4492 [1 << 17]byte +var z4493 [1 << 17]byte +var z4494 [1 << 17]byte +var z4495 [1 << 17]byte +var z4496 [1 << 17]byte +var z4497 [1 << 17]byte +var z4498 [1 << 17]byte +var z4499 [1 << 17]byte +var z4500 [1 << 17]byte +var z4501 [1 << 17]byte +var z4502 [1 << 17]byte +var z4503 [1 << 17]byte +var z4504 [1 << 17]byte +var z4505 [1 << 17]byte +var z4506 [1 << 17]byte +var z4507 [1 << 17]byte +var z4508 [1 << 17]byte +var z4509 [1 << 17]byte +var z4510 [1 << 17]byte +var z4511 [1 << 17]byte +var z4512 [1 << 17]byte +var z4513 [1 << 17]byte +var z4514 [1 << 17]byte +var z4515 [1 << 17]byte +var z4516 [1 << 17]byte +var z4517 [1 << 17]byte +var z4518 [1 << 17]byte +var z4519 [1 << 17]byte +var z4520 [1 << 17]byte +var z4521 [1 << 17]byte +var z4522 [1 << 17]byte +var z4523 [1 << 17]byte +var z4524 [1 << 17]byte +var z4525 [1 << 17]byte +var z4526 [1 << 17]byte +var z4527 [1 << 17]byte +var z4528 [1 << 17]byte +var z4529 [1 << 17]byte +var z4530 [1 << 17]byte +var z4531 [1 << 17]byte +var z4532 [1 << 17]byte +var z4533 [1 << 17]byte +var z4534 [1 << 17]byte +var z4535 [1 << 17]byte +var z4536 [1 << 17]byte +var z4537 [1 << 17]byte +var z4538 [1 << 17]byte +var z4539 [1 << 17]byte +var z4540 [1 << 17]byte +var z4541 [1 << 17]byte +var z4542 [1 << 17]byte +var z4543 [1 << 17]byte +var z4544 [1 << 17]byte +var z4545 [1 << 17]byte +var z4546 [1 << 17]byte +var z4547 [1 << 17]byte +var z4548 [1 << 17]byte +var z4549 [1 << 17]byte +var z4550 [1 << 17]byte +var z4551 [1 << 17]byte +var z4552 [1 << 17]byte +var z4553 [1 << 17]byte +var z4554 [1 << 17]byte +var z4555 [1 << 17]byte +var z4556 [1 << 17]byte +var z4557 [1 << 17]byte +var z4558 [1 << 17]byte +var z4559 [1 << 17]byte +var z4560 [1 << 17]byte +var z4561 [1 << 17]byte +var z4562 [1 << 17]byte +var z4563 [1 << 17]byte +var z4564 [1 << 17]byte +var z4565 [1 << 17]byte +var z4566 [1 << 17]byte +var z4567 [1 << 17]byte +var z4568 [1 << 17]byte +var z4569 [1 << 17]byte +var z4570 [1 << 17]byte +var z4571 [1 << 17]byte +var z4572 [1 << 17]byte +var z4573 [1 << 17]byte +var z4574 [1 << 17]byte +var z4575 [1 << 17]byte +var z4576 [1 << 17]byte +var z4577 [1 << 17]byte +var z4578 [1 << 17]byte +var z4579 [1 << 17]byte +var z4580 [1 << 17]byte +var z4581 [1 << 17]byte +var z4582 [1 << 17]byte +var z4583 [1 << 17]byte +var z4584 [1 << 17]byte +var z4585 [1 << 17]byte +var z4586 [1 << 17]byte +var z4587 [1 << 17]byte +var z4588 [1 << 17]byte +var z4589 [1 << 17]byte +var z4590 [1 << 17]byte +var z4591 [1 << 17]byte +var z4592 [1 << 17]byte +var z4593 [1 << 17]byte +var z4594 [1 << 17]byte +var z4595 [1 << 17]byte +var z4596 [1 << 17]byte +var z4597 [1 << 17]byte +var z4598 [1 << 17]byte +var z4599 [1 << 17]byte +var z4600 [1 << 17]byte +var z4601 [1 << 17]byte +var z4602 [1 << 17]byte +var z4603 [1 << 17]byte +var z4604 [1 << 17]byte +var z4605 [1 << 17]byte +var z4606 [1 << 17]byte +var z4607 [1 << 17]byte +var z4608 [1 << 17]byte +var z4609 [1 << 17]byte +var z4610 [1 << 17]byte +var z4611 [1 << 17]byte +var z4612 [1 << 17]byte +var z4613 [1 << 17]byte +var z4614 [1 << 17]byte +var z4615 [1 << 17]byte +var z4616 [1 << 17]byte +var z4617 [1 << 17]byte +var z4618 [1 << 17]byte +var z4619 [1 << 17]byte +var z4620 [1 << 17]byte +var z4621 [1 << 17]byte +var z4622 [1 << 17]byte +var z4623 [1 << 17]byte +var z4624 [1 << 17]byte +var z4625 [1 << 17]byte +var z4626 [1 << 17]byte +var z4627 [1 << 17]byte +var z4628 [1 << 17]byte +var z4629 [1 << 17]byte +var z4630 [1 << 17]byte +var z4631 [1 << 17]byte +var z4632 [1 << 17]byte +var z4633 [1 << 17]byte +var z4634 [1 << 17]byte +var z4635 [1 << 17]byte +var z4636 [1 << 17]byte +var z4637 [1 << 17]byte +var z4638 [1 << 17]byte +var z4639 [1 << 17]byte +var z4640 [1 << 17]byte +var z4641 [1 << 17]byte +var z4642 [1 << 17]byte +var z4643 [1 << 17]byte +var z4644 [1 << 17]byte +var z4645 [1 << 17]byte +var z4646 [1 << 17]byte +var z4647 [1 << 17]byte +var z4648 [1 << 17]byte +var z4649 [1 << 17]byte +var z4650 [1 << 17]byte +var z4651 [1 << 17]byte +var z4652 [1 << 17]byte +var z4653 [1 << 17]byte +var z4654 [1 << 17]byte +var z4655 [1 << 17]byte +var z4656 [1 << 17]byte +var z4657 [1 << 17]byte +var z4658 [1 << 17]byte +var z4659 [1 << 17]byte +var z4660 [1 << 17]byte +var z4661 [1 << 17]byte +var z4662 [1 << 17]byte +var z4663 [1 << 17]byte +var z4664 [1 << 17]byte +var z4665 [1 << 17]byte +var z4666 [1 << 17]byte +var z4667 [1 << 17]byte +var z4668 [1 << 17]byte +var z4669 [1 << 17]byte +var z4670 [1 << 17]byte +var z4671 [1 << 17]byte +var z4672 [1 << 17]byte +var z4673 [1 << 17]byte +var z4674 [1 << 17]byte +var z4675 [1 << 17]byte +var z4676 [1 << 17]byte +var z4677 [1 << 17]byte +var z4678 [1 << 17]byte +var z4679 [1 << 17]byte +var z4680 [1 << 17]byte +var z4681 [1 << 17]byte +var z4682 [1 << 17]byte +var z4683 [1 << 17]byte +var z4684 [1 << 17]byte +var z4685 [1 << 17]byte +var z4686 [1 << 17]byte +var z4687 [1 << 17]byte +var z4688 [1 << 17]byte +var z4689 [1 << 17]byte +var z4690 [1 << 17]byte +var z4691 [1 << 17]byte +var z4692 [1 << 17]byte +var z4693 [1 << 17]byte +var z4694 [1 << 17]byte +var z4695 [1 << 17]byte +var z4696 [1 << 17]byte +var z4697 [1 << 17]byte +var z4698 [1 << 17]byte +var z4699 [1 << 17]byte +var z4700 [1 << 17]byte +var z4701 [1 << 17]byte +var z4702 [1 << 17]byte +var z4703 [1 << 17]byte +var z4704 [1 << 17]byte +var z4705 [1 << 17]byte +var z4706 [1 << 17]byte +var z4707 [1 << 17]byte +var z4708 [1 << 17]byte +var z4709 [1 << 17]byte +var z4710 [1 << 17]byte +var z4711 [1 << 17]byte +var z4712 [1 << 17]byte +var z4713 [1 << 17]byte +var z4714 [1 << 17]byte +var z4715 [1 << 17]byte +var z4716 [1 << 17]byte +var z4717 [1 << 17]byte +var z4718 [1 << 17]byte +var z4719 [1 << 17]byte +var z4720 [1 << 17]byte +var z4721 [1 << 17]byte +var z4722 [1 << 17]byte +var z4723 [1 << 17]byte +var z4724 [1 << 17]byte +var z4725 [1 << 17]byte +var z4726 [1 << 17]byte +var z4727 [1 << 17]byte +var z4728 [1 << 17]byte +var z4729 [1 << 17]byte +var z4730 [1 << 17]byte +var z4731 [1 << 17]byte +var z4732 [1 << 17]byte +var z4733 [1 << 17]byte +var z4734 [1 << 17]byte +var z4735 [1 << 17]byte +var z4736 [1 << 17]byte +var z4737 [1 << 17]byte +var z4738 [1 << 17]byte +var z4739 [1 << 17]byte +var z4740 [1 << 17]byte +var z4741 [1 << 17]byte +var z4742 [1 << 17]byte +var z4743 [1 << 17]byte +var z4744 [1 << 17]byte +var z4745 [1 << 17]byte +var z4746 [1 << 17]byte +var z4747 [1 << 17]byte +var z4748 [1 << 17]byte +var z4749 [1 << 17]byte +var z4750 [1 << 17]byte +var z4751 [1 << 17]byte +var z4752 [1 << 17]byte +var z4753 [1 << 17]byte +var z4754 [1 << 17]byte +var z4755 [1 << 17]byte +var z4756 [1 << 17]byte +var z4757 [1 << 17]byte +var z4758 [1 << 17]byte +var z4759 [1 << 17]byte +var z4760 [1 << 17]byte +var z4761 [1 << 17]byte +var z4762 [1 << 17]byte +var z4763 [1 << 17]byte +var z4764 [1 << 17]byte +var z4765 [1 << 17]byte +var z4766 [1 << 17]byte +var z4767 [1 << 17]byte +var z4768 [1 << 17]byte +var z4769 [1 << 17]byte +var z4770 [1 << 17]byte +var z4771 [1 << 17]byte +var z4772 [1 << 17]byte +var z4773 [1 << 17]byte +var z4774 [1 << 17]byte +var z4775 [1 << 17]byte +var z4776 [1 << 17]byte +var z4777 [1 << 17]byte +var z4778 [1 << 17]byte +var z4779 [1 << 17]byte +var z4780 [1 << 17]byte +var z4781 [1 << 17]byte +var z4782 [1 << 17]byte +var z4783 [1 << 17]byte +var z4784 [1 << 17]byte +var z4785 [1 << 17]byte +var z4786 [1 << 17]byte +var z4787 [1 << 17]byte +var z4788 [1 << 17]byte +var z4789 [1 << 17]byte +var z4790 [1 << 17]byte +var z4791 [1 << 17]byte +var z4792 [1 << 17]byte +var z4793 [1 << 17]byte +var z4794 [1 << 17]byte +var z4795 [1 << 17]byte +var z4796 [1 << 17]byte +var z4797 [1 << 17]byte +var z4798 [1 << 17]byte +var z4799 [1 << 17]byte +var z4800 [1 << 17]byte +var z4801 [1 << 17]byte +var z4802 [1 << 17]byte +var z4803 [1 << 17]byte +var z4804 [1 << 17]byte +var z4805 [1 << 17]byte +var z4806 [1 << 17]byte +var z4807 [1 << 17]byte +var z4808 [1 << 17]byte +var z4809 [1 << 17]byte +var z4810 [1 << 17]byte +var z4811 [1 << 17]byte +var z4812 [1 << 17]byte +var z4813 [1 << 17]byte +var z4814 [1 << 17]byte +var z4815 [1 << 17]byte +var z4816 [1 << 17]byte +var z4817 [1 << 17]byte +var z4818 [1 << 17]byte +var z4819 [1 << 17]byte +var z4820 [1 << 17]byte +var z4821 [1 << 17]byte +var z4822 [1 << 17]byte +var z4823 [1 << 17]byte +var z4824 [1 << 17]byte +var z4825 [1 << 17]byte +var z4826 [1 << 17]byte +var z4827 [1 << 17]byte +var z4828 [1 << 17]byte +var z4829 [1 << 17]byte +var z4830 [1 << 17]byte +var z4831 [1 << 17]byte +var z4832 [1 << 17]byte +var z4833 [1 << 17]byte +var z4834 [1 << 17]byte +var z4835 [1 << 17]byte +var z4836 [1 << 17]byte +var z4837 [1 << 17]byte +var z4838 [1 << 17]byte +var z4839 [1 << 17]byte +var z4840 [1 << 17]byte +var z4841 [1 << 17]byte +var z4842 [1 << 17]byte +var z4843 [1 << 17]byte +var z4844 [1 << 17]byte +var z4845 [1 << 17]byte +var z4846 [1 << 17]byte +var z4847 [1 << 17]byte +var z4848 [1 << 17]byte +var z4849 [1 << 17]byte +var z4850 [1 << 17]byte +var z4851 [1 << 17]byte +var z4852 [1 << 17]byte +var z4853 [1 << 17]byte +var z4854 [1 << 17]byte +var z4855 [1 << 17]byte +var z4856 [1 << 17]byte +var z4857 [1 << 17]byte +var z4858 [1 << 17]byte +var z4859 [1 << 17]byte +var z4860 [1 << 17]byte +var z4861 [1 << 17]byte +var z4862 [1 << 17]byte +var z4863 [1 << 17]byte +var z4864 [1 << 17]byte +var z4865 [1 << 17]byte +var z4866 [1 << 17]byte +var z4867 [1 << 17]byte +var z4868 [1 << 17]byte +var z4869 [1 << 17]byte +var z4870 [1 << 17]byte +var z4871 [1 << 17]byte +var z4872 [1 << 17]byte +var z4873 [1 << 17]byte +var z4874 [1 << 17]byte +var z4875 [1 << 17]byte +var z4876 [1 << 17]byte +var z4877 [1 << 17]byte +var z4878 [1 << 17]byte +var z4879 [1 << 17]byte +var z4880 [1 << 17]byte +var z4881 [1 << 17]byte +var z4882 [1 << 17]byte +var z4883 [1 << 17]byte +var z4884 [1 << 17]byte +var z4885 [1 << 17]byte +var z4886 [1 << 17]byte +var z4887 [1 << 17]byte +var z4888 [1 << 17]byte +var z4889 [1 << 17]byte +var z4890 [1 << 17]byte +var z4891 [1 << 17]byte +var z4892 [1 << 17]byte +var z4893 [1 << 17]byte +var z4894 [1 << 17]byte +var z4895 [1 << 17]byte +var z4896 [1 << 17]byte +var z4897 [1 << 17]byte +var z4898 [1 << 17]byte +var z4899 [1 << 17]byte +var z4900 [1 << 17]byte +var z4901 [1 << 17]byte +var z4902 [1 << 17]byte +var z4903 [1 << 17]byte +var z4904 [1 << 17]byte +var z4905 [1 << 17]byte +var z4906 [1 << 17]byte +var z4907 [1 << 17]byte +var z4908 [1 << 17]byte +var z4909 [1 << 17]byte +var z4910 [1 << 17]byte +var z4911 [1 << 17]byte +var z4912 [1 << 17]byte +var z4913 [1 << 17]byte +var z4914 [1 << 17]byte +var z4915 [1 << 17]byte +var z4916 [1 << 17]byte +var z4917 [1 << 17]byte +var z4918 [1 << 17]byte +var z4919 [1 << 17]byte +var z4920 [1 << 17]byte +var z4921 [1 << 17]byte +var z4922 [1 << 17]byte +var z4923 [1 << 17]byte +var z4924 [1 << 17]byte +var z4925 [1 << 17]byte +var z4926 [1 << 17]byte +var z4927 [1 << 17]byte +var z4928 [1 << 17]byte +var z4929 [1 << 17]byte +var z4930 [1 << 17]byte +var z4931 [1 << 17]byte +var z4932 [1 << 17]byte +var z4933 [1 << 17]byte +var z4934 [1 << 17]byte +var z4935 [1 << 17]byte +var z4936 [1 << 17]byte +var z4937 [1 << 17]byte +var z4938 [1 << 17]byte +var z4939 [1 << 17]byte +var z4940 [1 << 17]byte +var z4941 [1 << 17]byte +var z4942 [1 << 17]byte +var z4943 [1 << 17]byte +var z4944 [1 << 17]byte +var z4945 [1 << 17]byte +var z4946 [1 << 17]byte +var z4947 [1 << 17]byte +var z4948 [1 << 17]byte +var z4949 [1 << 17]byte +var z4950 [1 << 17]byte +var z4951 [1 << 17]byte +var z4952 [1 << 17]byte +var z4953 [1 << 17]byte +var z4954 [1 << 17]byte +var z4955 [1 << 17]byte +var z4956 [1 << 17]byte +var z4957 [1 << 17]byte +var z4958 [1 << 17]byte +var z4959 [1 << 17]byte +var z4960 [1 << 17]byte +var z4961 [1 << 17]byte +var z4962 [1 << 17]byte +var z4963 [1 << 17]byte +var z4964 [1 << 17]byte +var z4965 [1 << 17]byte +var z4966 [1 << 17]byte +var z4967 [1 << 17]byte +var z4968 [1 << 17]byte +var z4969 [1 << 17]byte +var z4970 [1 << 17]byte +var z4971 [1 << 17]byte +var z4972 [1 << 17]byte +var z4973 [1 << 17]byte +var z4974 [1 << 17]byte +var z4975 [1 << 17]byte +var z4976 [1 << 17]byte +var z4977 [1 << 17]byte +var z4978 [1 << 17]byte +var z4979 [1 << 17]byte +var z4980 [1 << 17]byte +var z4981 [1 << 17]byte +var z4982 [1 << 17]byte +var z4983 [1 << 17]byte +var z4984 [1 << 17]byte +var z4985 [1 << 17]byte +var z4986 [1 << 17]byte +var z4987 [1 << 17]byte +var z4988 [1 << 17]byte +var z4989 [1 << 17]byte +var z4990 [1 << 17]byte +var z4991 [1 << 17]byte +var z4992 [1 << 17]byte +var z4993 [1 << 17]byte +var z4994 [1 << 17]byte +var z4995 [1 << 17]byte +var z4996 [1 << 17]byte +var z4997 [1 << 17]byte +var z4998 [1 << 17]byte +var z4999 [1 << 17]byte +var z5000 [1 << 17]byte +var z5001 [1 << 17]byte +var z5002 [1 << 17]byte +var z5003 [1 << 17]byte +var z5004 [1 << 17]byte +var z5005 [1 << 17]byte +var z5006 [1 << 17]byte +var z5007 [1 << 17]byte +var z5008 [1 << 17]byte +var z5009 [1 << 17]byte +var z5010 [1 << 17]byte +var z5011 [1 << 17]byte +var z5012 [1 << 17]byte +var z5013 [1 << 17]byte +var z5014 [1 << 17]byte +var z5015 [1 << 17]byte +var z5016 [1 << 17]byte +var z5017 [1 << 17]byte +var z5018 [1 << 17]byte +var z5019 [1 << 17]byte +var z5020 [1 << 17]byte +var z5021 [1 << 17]byte +var z5022 [1 << 17]byte +var z5023 [1 << 17]byte +var z5024 [1 << 17]byte +var z5025 [1 << 17]byte +var z5026 [1 << 17]byte +var z5027 [1 << 17]byte +var z5028 [1 << 17]byte +var z5029 [1 << 17]byte +var z5030 [1 << 17]byte +var z5031 [1 << 17]byte +var z5032 [1 << 17]byte +var z5033 [1 << 17]byte +var z5034 [1 << 17]byte +var z5035 [1 << 17]byte +var z5036 [1 << 17]byte +var z5037 [1 << 17]byte +var z5038 [1 << 17]byte +var z5039 [1 << 17]byte +var z5040 [1 << 17]byte +var z5041 [1 << 17]byte +var z5042 [1 << 17]byte +var z5043 [1 << 17]byte +var z5044 [1 << 17]byte +var z5045 [1 << 17]byte +var z5046 [1 << 17]byte +var z5047 [1 << 17]byte +var z5048 [1 << 17]byte +var z5049 [1 << 17]byte +var z5050 [1 << 17]byte +var z5051 [1 << 17]byte +var z5052 [1 << 17]byte +var z5053 [1 << 17]byte +var z5054 [1 << 17]byte +var z5055 [1 << 17]byte +var z5056 [1 << 17]byte +var z5057 [1 << 17]byte +var z5058 [1 << 17]byte +var z5059 [1 << 17]byte +var z5060 [1 << 17]byte +var z5061 [1 << 17]byte +var z5062 [1 << 17]byte +var z5063 [1 << 17]byte +var z5064 [1 << 17]byte +var z5065 [1 << 17]byte +var z5066 [1 << 17]byte +var z5067 [1 << 17]byte +var z5068 [1 << 17]byte +var z5069 [1 << 17]byte +var z5070 [1 << 17]byte +var z5071 [1 << 17]byte +var z5072 [1 << 17]byte +var z5073 [1 << 17]byte +var z5074 [1 << 17]byte +var z5075 [1 << 17]byte +var z5076 [1 << 17]byte +var z5077 [1 << 17]byte +var z5078 [1 << 17]byte +var z5079 [1 << 17]byte +var z5080 [1 << 17]byte +var z5081 [1 << 17]byte +var z5082 [1 << 17]byte +var z5083 [1 << 17]byte +var z5084 [1 << 17]byte +var z5085 [1 << 17]byte +var z5086 [1 << 17]byte +var z5087 [1 << 17]byte +var z5088 [1 << 17]byte +var z5089 [1 << 17]byte +var z5090 [1 << 17]byte +var z5091 [1 << 17]byte +var z5092 [1 << 17]byte +var z5093 [1 << 17]byte +var z5094 [1 << 17]byte +var z5095 [1 << 17]byte +var z5096 [1 << 17]byte +var z5097 [1 << 17]byte +var z5098 [1 << 17]byte +var z5099 [1 << 17]byte +var z5100 [1 << 17]byte +var z5101 [1 << 17]byte +var z5102 [1 << 17]byte +var z5103 [1 << 17]byte +var z5104 [1 << 17]byte +var z5105 [1 << 17]byte +var z5106 [1 << 17]byte +var z5107 [1 << 17]byte +var z5108 [1 << 17]byte +var z5109 [1 << 17]byte +var z5110 [1 << 17]byte +var z5111 [1 << 17]byte +var z5112 [1 << 17]byte +var z5113 [1 << 17]byte +var z5114 [1 << 17]byte +var z5115 [1 << 17]byte +var z5116 [1 << 17]byte +var z5117 [1 << 17]byte +var z5118 [1 << 17]byte +var z5119 [1 << 17]byte +var z5120 [1 << 17]byte +var z5121 [1 << 17]byte +var z5122 [1 << 17]byte +var z5123 [1 << 17]byte +var z5124 [1 << 17]byte +var z5125 [1 << 17]byte +var z5126 [1 << 17]byte +var z5127 [1 << 17]byte +var z5128 [1 << 17]byte +var z5129 [1 << 17]byte +var z5130 [1 << 17]byte +var z5131 [1 << 17]byte +var z5132 [1 << 17]byte +var z5133 [1 << 17]byte +var z5134 [1 << 17]byte +var z5135 [1 << 17]byte +var z5136 [1 << 17]byte +var z5137 [1 << 17]byte +var z5138 [1 << 17]byte +var z5139 [1 << 17]byte +var z5140 [1 << 17]byte +var z5141 [1 << 17]byte +var z5142 [1 << 17]byte +var z5143 [1 << 17]byte +var z5144 [1 << 17]byte +var z5145 [1 << 17]byte +var z5146 [1 << 17]byte +var z5147 [1 << 17]byte +var z5148 [1 << 17]byte +var z5149 [1 << 17]byte +var z5150 [1 << 17]byte +var z5151 [1 << 17]byte +var z5152 [1 << 17]byte +var z5153 [1 << 17]byte +var z5154 [1 << 17]byte +var z5155 [1 << 17]byte +var z5156 [1 << 17]byte +var z5157 [1 << 17]byte +var z5158 [1 << 17]byte +var z5159 [1 << 17]byte +var z5160 [1 << 17]byte +var z5161 [1 << 17]byte +var z5162 [1 << 17]byte +var z5163 [1 << 17]byte +var z5164 [1 << 17]byte +var z5165 [1 << 17]byte +var z5166 [1 << 17]byte +var z5167 [1 << 17]byte +var z5168 [1 << 17]byte +var z5169 [1 << 17]byte +var z5170 [1 << 17]byte +var z5171 [1 << 17]byte +var z5172 [1 << 17]byte +var z5173 [1 << 17]byte +var z5174 [1 << 17]byte +var z5175 [1 << 17]byte +var z5176 [1 << 17]byte +var z5177 [1 << 17]byte +var z5178 [1 << 17]byte +var z5179 [1 << 17]byte +var z5180 [1 << 17]byte +var z5181 [1 << 17]byte +var z5182 [1 << 17]byte +var z5183 [1 << 17]byte +var z5184 [1 << 17]byte +var z5185 [1 << 17]byte +var z5186 [1 << 17]byte +var z5187 [1 << 17]byte +var z5188 [1 << 17]byte +var z5189 [1 << 17]byte +var z5190 [1 << 17]byte +var z5191 [1 << 17]byte +var z5192 [1 << 17]byte +var z5193 [1 << 17]byte +var z5194 [1 << 17]byte +var z5195 [1 << 17]byte +var z5196 [1 << 17]byte +var z5197 [1 << 17]byte +var z5198 [1 << 17]byte +var z5199 [1 << 17]byte +var z5200 [1 << 17]byte +var z5201 [1 << 17]byte +var z5202 [1 << 17]byte +var z5203 [1 << 17]byte +var z5204 [1 << 17]byte +var z5205 [1 << 17]byte +var z5206 [1 << 17]byte +var z5207 [1 << 17]byte +var z5208 [1 << 17]byte +var z5209 [1 << 17]byte +var z5210 [1 << 17]byte +var z5211 [1 << 17]byte +var z5212 [1 << 17]byte +var z5213 [1 << 17]byte +var z5214 [1 << 17]byte +var z5215 [1 << 17]byte +var z5216 [1 << 17]byte +var z5217 [1 << 17]byte +var z5218 [1 << 17]byte +var z5219 [1 << 17]byte +var z5220 [1 << 17]byte +var z5221 [1 << 17]byte +var z5222 [1 << 17]byte +var z5223 [1 << 17]byte +var z5224 [1 << 17]byte +var z5225 [1 << 17]byte +var z5226 [1 << 17]byte +var z5227 [1 << 17]byte +var z5228 [1 << 17]byte +var z5229 [1 << 17]byte +var z5230 [1 << 17]byte +var z5231 [1 << 17]byte +var z5232 [1 << 17]byte +var z5233 [1 << 17]byte +var z5234 [1 << 17]byte +var z5235 [1 << 17]byte +var z5236 [1 << 17]byte +var z5237 [1 << 17]byte +var z5238 [1 << 17]byte +var z5239 [1 << 17]byte +var z5240 [1 << 17]byte +var z5241 [1 << 17]byte +var z5242 [1 << 17]byte +var z5243 [1 << 17]byte +var z5244 [1 << 17]byte +var z5245 [1 << 17]byte +var z5246 [1 << 17]byte +var z5247 [1 << 17]byte +var z5248 [1 << 17]byte +var z5249 [1 << 17]byte +var z5250 [1 << 17]byte +var z5251 [1 << 17]byte +var z5252 [1 << 17]byte +var z5253 [1 << 17]byte +var z5254 [1 << 17]byte +var z5255 [1 << 17]byte +var z5256 [1 << 17]byte +var z5257 [1 << 17]byte +var z5258 [1 << 17]byte +var z5259 [1 << 17]byte +var z5260 [1 << 17]byte +var z5261 [1 << 17]byte +var z5262 [1 << 17]byte +var z5263 [1 << 17]byte +var z5264 [1 << 17]byte +var z5265 [1 << 17]byte +var z5266 [1 << 17]byte +var z5267 [1 << 17]byte +var z5268 [1 << 17]byte +var z5269 [1 << 17]byte +var z5270 [1 << 17]byte +var z5271 [1 << 17]byte +var z5272 [1 << 17]byte +var z5273 [1 << 17]byte +var z5274 [1 << 17]byte +var z5275 [1 << 17]byte +var z5276 [1 << 17]byte +var z5277 [1 << 17]byte +var z5278 [1 << 17]byte +var z5279 [1 << 17]byte +var z5280 [1 << 17]byte +var z5281 [1 << 17]byte +var z5282 [1 << 17]byte +var z5283 [1 << 17]byte +var z5284 [1 << 17]byte +var z5285 [1 << 17]byte +var z5286 [1 << 17]byte +var z5287 [1 << 17]byte +var z5288 [1 << 17]byte +var z5289 [1 << 17]byte +var z5290 [1 << 17]byte +var z5291 [1 << 17]byte +var z5292 [1 << 17]byte +var z5293 [1 << 17]byte +var z5294 [1 << 17]byte +var z5295 [1 << 17]byte +var z5296 [1 << 17]byte +var z5297 [1 << 17]byte +var z5298 [1 << 17]byte +var z5299 [1 << 17]byte +var z5300 [1 << 17]byte +var z5301 [1 << 17]byte +var z5302 [1 << 17]byte +var z5303 [1 << 17]byte +var z5304 [1 << 17]byte +var z5305 [1 << 17]byte +var z5306 [1 << 17]byte +var z5307 [1 << 17]byte +var z5308 [1 << 17]byte +var z5309 [1 << 17]byte +var z5310 [1 << 17]byte +var z5311 [1 << 17]byte +var z5312 [1 << 17]byte +var z5313 [1 << 17]byte +var z5314 [1 << 17]byte +var z5315 [1 << 17]byte +var z5316 [1 << 17]byte +var z5317 [1 << 17]byte +var z5318 [1 << 17]byte +var z5319 [1 << 17]byte +var z5320 [1 << 17]byte +var z5321 [1 << 17]byte +var z5322 [1 << 17]byte +var z5323 [1 << 17]byte +var z5324 [1 << 17]byte +var z5325 [1 << 17]byte +var z5326 [1 << 17]byte +var z5327 [1 << 17]byte +var z5328 [1 << 17]byte +var z5329 [1 << 17]byte +var z5330 [1 << 17]byte +var z5331 [1 << 17]byte +var z5332 [1 << 17]byte +var z5333 [1 << 17]byte +var z5334 [1 << 17]byte +var z5335 [1 << 17]byte +var z5336 [1 << 17]byte +var z5337 [1 << 17]byte +var z5338 [1 << 17]byte +var z5339 [1 << 17]byte +var z5340 [1 << 17]byte +var z5341 [1 << 17]byte +var z5342 [1 << 17]byte +var z5343 [1 << 17]byte +var z5344 [1 << 17]byte +var z5345 [1 << 17]byte +var z5346 [1 << 17]byte +var z5347 [1 << 17]byte +var z5348 [1 << 17]byte +var z5349 [1 << 17]byte +var z5350 [1 << 17]byte +var z5351 [1 << 17]byte +var z5352 [1 << 17]byte +var z5353 [1 << 17]byte +var z5354 [1 << 17]byte +var z5355 [1 << 17]byte +var z5356 [1 << 17]byte +var z5357 [1 << 17]byte +var z5358 [1 << 17]byte +var z5359 [1 << 17]byte +var z5360 [1 << 17]byte +var z5361 [1 << 17]byte +var z5362 [1 << 17]byte +var z5363 [1 << 17]byte +var z5364 [1 << 17]byte +var z5365 [1 << 17]byte +var z5366 [1 << 17]byte +var z5367 [1 << 17]byte +var z5368 [1 << 17]byte +var z5369 [1 << 17]byte +var z5370 [1 << 17]byte +var z5371 [1 << 17]byte +var z5372 [1 << 17]byte +var z5373 [1 << 17]byte +var z5374 [1 << 17]byte +var z5375 [1 << 17]byte +var z5376 [1 << 17]byte +var z5377 [1 << 17]byte +var z5378 [1 << 17]byte +var z5379 [1 << 17]byte +var z5380 [1 << 17]byte +var z5381 [1 << 17]byte +var z5382 [1 << 17]byte +var z5383 [1 << 17]byte +var z5384 [1 << 17]byte +var z5385 [1 << 17]byte +var z5386 [1 << 17]byte +var z5387 [1 << 17]byte +var z5388 [1 << 17]byte +var z5389 [1 << 17]byte +var z5390 [1 << 17]byte +var z5391 [1 << 17]byte +var z5392 [1 << 17]byte +var z5393 [1 << 17]byte +var z5394 [1 << 17]byte +var z5395 [1 << 17]byte +var z5396 [1 << 17]byte +var z5397 [1 << 17]byte +var z5398 [1 << 17]byte +var z5399 [1 << 17]byte +var z5400 [1 << 17]byte +var z5401 [1 << 17]byte +var z5402 [1 << 17]byte +var z5403 [1 << 17]byte +var z5404 [1 << 17]byte +var z5405 [1 << 17]byte +var z5406 [1 << 17]byte +var z5407 [1 << 17]byte +var z5408 [1 << 17]byte +var z5409 [1 << 17]byte +var z5410 [1 << 17]byte +var z5411 [1 << 17]byte +var z5412 [1 << 17]byte +var z5413 [1 << 17]byte +var z5414 [1 << 17]byte +var z5415 [1 << 17]byte +var z5416 [1 << 17]byte +var z5417 [1 << 17]byte +var z5418 [1 << 17]byte +var z5419 [1 << 17]byte +var z5420 [1 << 17]byte +var z5421 [1 << 17]byte +var z5422 [1 << 17]byte +var z5423 [1 << 17]byte +var z5424 [1 << 17]byte +var z5425 [1 << 17]byte +var z5426 [1 << 17]byte +var z5427 [1 << 17]byte +var z5428 [1 << 17]byte +var z5429 [1 << 17]byte +var z5430 [1 << 17]byte +var z5431 [1 << 17]byte +var z5432 [1 << 17]byte +var z5433 [1 << 17]byte +var z5434 [1 << 17]byte +var z5435 [1 << 17]byte +var z5436 [1 << 17]byte +var z5437 [1 << 17]byte +var z5438 [1 << 17]byte +var z5439 [1 << 17]byte +var z5440 [1 << 17]byte +var z5441 [1 << 17]byte +var z5442 [1 << 17]byte +var z5443 [1 << 17]byte +var z5444 [1 << 17]byte +var z5445 [1 << 17]byte +var z5446 [1 << 17]byte +var z5447 [1 << 17]byte +var z5448 [1 << 17]byte +var z5449 [1 << 17]byte +var z5450 [1 << 17]byte +var z5451 [1 << 17]byte +var z5452 [1 << 17]byte +var z5453 [1 << 17]byte +var z5454 [1 << 17]byte +var z5455 [1 << 17]byte +var z5456 [1 << 17]byte +var z5457 [1 << 17]byte +var z5458 [1 << 17]byte +var z5459 [1 << 17]byte +var z5460 [1 << 17]byte +var z5461 [1 << 17]byte +var z5462 [1 << 17]byte +var z5463 [1 << 17]byte +var z5464 [1 << 17]byte +var z5465 [1 << 17]byte +var z5466 [1 << 17]byte +var z5467 [1 << 17]byte +var z5468 [1 << 17]byte +var z5469 [1 << 17]byte +var z5470 [1 << 17]byte +var z5471 [1 << 17]byte +var z5472 [1 << 17]byte +var z5473 [1 << 17]byte +var z5474 [1 << 17]byte +var z5475 [1 << 17]byte +var z5476 [1 << 17]byte +var z5477 [1 << 17]byte +var z5478 [1 << 17]byte +var z5479 [1 << 17]byte +var z5480 [1 << 17]byte +var z5481 [1 << 17]byte +var z5482 [1 << 17]byte +var z5483 [1 << 17]byte +var z5484 [1 << 17]byte +var z5485 [1 << 17]byte +var z5486 [1 << 17]byte +var z5487 [1 << 17]byte +var z5488 [1 << 17]byte +var z5489 [1 << 17]byte +var z5490 [1 << 17]byte +var z5491 [1 << 17]byte +var z5492 [1 << 17]byte +var z5493 [1 << 17]byte +var z5494 [1 << 17]byte +var z5495 [1 << 17]byte +var z5496 [1 << 17]byte +var z5497 [1 << 17]byte +var z5498 [1 << 17]byte +var z5499 [1 << 17]byte +var z5500 [1 << 17]byte +var z5501 [1 << 17]byte +var z5502 [1 << 17]byte +var z5503 [1 << 17]byte +var z5504 [1 << 17]byte +var z5505 [1 << 17]byte +var z5506 [1 << 17]byte +var z5507 [1 << 17]byte +var z5508 [1 << 17]byte +var z5509 [1 << 17]byte +var z5510 [1 << 17]byte +var z5511 [1 << 17]byte +var z5512 [1 << 17]byte +var z5513 [1 << 17]byte +var z5514 [1 << 17]byte +var z5515 [1 << 17]byte +var z5516 [1 << 17]byte +var z5517 [1 << 17]byte +var z5518 [1 << 17]byte +var z5519 [1 << 17]byte +var z5520 [1 << 17]byte +var z5521 [1 << 17]byte +var z5522 [1 << 17]byte +var z5523 [1 << 17]byte +var z5524 [1 << 17]byte +var z5525 [1 << 17]byte +var z5526 [1 << 17]byte +var z5527 [1 << 17]byte +var z5528 [1 << 17]byte +var z5529 [1 << 17]byte +var z5530 [1 << 17]byte +var z5531 [1 << 17]byte +var z5532 [1 << 17]byte +var z5533 [1 << 17]byte +var z5534 [1 << 17]byte +var z5535 [1 << 17]byte +var z5536 [1 << 17]byte +var z5537 [1 << 17]byte +var z5538 [1 << 17]byte +var z5539 [1 << 17]byte +var z5540 [1 << 17]byte +var z5541 [1 << 17]byte +var z5542 [1 << 17]byte +var z5543 [1 << 17]byte +var z5544 [1 << 17]byte +var z5545 [1 << 17]byte +var z5546 [1 << 17]byte +var z5547 [1 << 17]byte +var z5548 [1 << 17]byte +var z5549 [1 << 17]byte +var z5550 [1 << 17]byte +var z5551 [1 << 17]byte +var z5552 [1 << 17]byte +var z5553 [1 << 17]byte +var z5554 [1 << 17]byte +var z5555 [1 << 17]byte +var z5556 [1 << 17]byte +var z5557 [1 << 17]byte +var z5558 [1 << 17]byte +var z5559 [1 << 17]byte +var z5560 [1 << 17]byte +var z5561 [1 << 17]byte +var z5562 [1 << 17]byte +var z5563 [1 << 17]byte +var z5564 [1 << 17]byte +var z5565 [1 << 17]byte +var z5566 [1 << 17]byte +var z5567 [1 << 17]byte +var z5568 [1 << 17]byte +var z5569 [1 << 17]byte +var z5570 [1 << 17]byte +var z5571 [1 << 17]byte +var z5572 [1 << 17]byte +var z5573 [1 << 17]byte +var z5574 [1 << 17]byte +var z5575 [1 << 17]byte +var z5576 [1 << 17]byte +var z5577 [1 << 17]byte +var z5578 [1 << 17]byte +var z5579 [1 << 17]byte +var z5580 [1 << 17]byte +var z5581 [1 << 17]byte +var z5582 [1 << 17]byte +var z5583 [1 << 17]byte +var z5584 [1 << 17]byte +var z5585 [1 << 17]byte +var z5586 [1 << 17]byte +var z5587 [1 << 17]byte +var z5588 [1 << 17]byte +var z5589 [1 << 17]byte +var z5590 [1 << 17]byte +var z5591 [1 << 17]byte +var z5592 [1 << 17]byte +var z5593 [1 << 17]byte +var z5594 [1 << 17]byte +var z5595 [1 << 17]byte +var z5596 [1 << 17]byte +var z5597 [1 << 17]byte +var z5598 [1 << 17]byte +var z5599 [1 << 17]byte +var z5600 [1 << 17]byte +var z5601 [1 << 17]byte +var z5602 [1 << 17]byte +var z5603 [1 << 17]byte +var z5604 [1 << 17]byte +var z5605 [1 << 17]byte +var z5606 [1 << 17]byte +var z5607 [1 << 17]byte +var z5608 [1 << 17]byte +var z5609 [1 << 17]byte +var z5610 [1 << 17]byte +var z5611 [1 << 17]byte +var z5612 [1 << 17]byte +var z5613 [1 << 17]byte +var z5614 [1 << 17]byte +var z5615 [1 << 17]byte +var z5616 [1 << 17]byte +var z5617 [1 << 17]byte +var z5618 [1 << 17]byte +var z5619 [1 << 17]byte +var z5620 [1 << 17]byte +var z5621 [1 << 17]byte +var z5622 [1 << 17]byte +var z5623 [1 << 17]byte +var z5624 [1 << 17]byte +var z5625 [1 << 17]byte +var z5626 [1 << 17]byte +var z5627 [1 << 17]byte +var z5628 [1 << 17]byte +var z5629 [1 << 17]byte +var z5630 [1 << 17]byte +var z5631 [1 << 17]byte +var z5632 [1 << 17]byte +var z5633 [1 << 17]byte +var z5634 [1 << 17]byte +var z5635 [1 << 17]byte +var z5636 [1 << 17]byte +var z5637 [1 << 17]byte +var z5638 [1 << 17]byte +var z5639 [1 << 17]byte +var z5640 [1 << 17]byte +var z5641 [1 << 17]byte +var z5642 [1 << 17]byte +var z5643 [1 << 17]byte +var z5644 [1 << 17]byte +var z5645 [1 << 17]byte +var z5646 [1 << 17]byte +var z5647 [1 << 17]byte +var z5648 [1 << 17]byte +var z5649 [1 << 17]byte +var z5650 [1 << 17]byte +var z5651 [1 << 17]byte +var z5652 [1 << 17]byte +var z5653 [1 << 17]byte +var z5654 [1 << 17]byte +var z5655 [1 << 17]byte +var z5656 [1 << 17]byte +var z5657 [1 << 17]byte +var z5658 [1 << 17]byte +var z5659 [1 << 17]byte +var z5660 [1 << 17]byte +var z5661 [1 << 17]byte +var z5662 [1 << 17]byte +var z5663 [1 << 17]byte +var z5664 [1 << 17]byte +var z5665 [1 << 17]byte +var z5666 [1 << 17]byte +var z5667 [1 << 17]byte +var z5668 [1 << 17]byte +var z5669 [1 << 17]byte +var z5670 [1 << 17]byte +var z5671 [1 << 17]byte +var z5672 [1 << 17]byte +var z5673 [1 << 17]byte +var z5674 [1 << 17]byte +var z5675 [1 << 17]byte +var z5676 [1 << 17]byte +var z5677 [1 << 17]byte +var z5678 [1 << 17]byte +var z5679 [1 << 17]byte +var z5680 [1 << 17]byte +var z5681 [1 << 17]byte +var z5682 [1 << 17]byte +var z5683 [1 << 17]byte +var z5684 [1 << 17]byte +var z5685 [1 << 17]byte +var z5686 [1 << 17]byte +var z5687 [1 << 17]byte +var z5688 [1 << 17]byte +var z5689 [1 << 17]byte +var z5690 [1 << 17]byte +var z5691 [1 << 17]byte +var z5692 [1 << 17]byte +var z5693 [1 << 17]byte +var z5694 [1 << 17]byte +var z5695 [1 << 17]byte +var z5696 [1 << 17]byte +var z5697 [1 << 17]byte +var z5698 [1 << 17]byte +var z5699 [1 << 17]byte +var z5700 [1 << 17]byte +var z5701 [1 << 17]byte +var z5702 [1 << 17]byte +var z5703 [1 << 17]byte +var z5704 [1 << 17]byte +var z5705 [1 << 17]byte +var z5706 [1 << 17]byte +var z5707 [1 << 17]byte +var z5708 [1 << 17]byte +var z5709 [1 << 17]byte +var z5710 [1 << 17]byte +var z5711 [1 << 17]byte +var z5712 [1 << 17]byte +var z5713 [1 << 17]byte +var z5714 [1 << 17]byte +var z5715 [1 << 17]byte +var z5716 [1 << 17]byte +var z5717 [1 << 17]byte +var z5718 [1 << 17]byte +var z5719 [1 << 17]byte +var z5720 [1 << 17]byte +var z5721 [1 << 17]byte +var z5722 [1 << 17]byte +var z5723 [1 << 17]byte +var z5724 [1 << 17]byte +var z5725 [1 << 17]byte +var z5726 [1 << 17]byte +var z5727 [1 << 17]byte +var z5728 [1 << 17]byte +var z5729 [1 << 17]byte +var z5730 [1 << 17]byte +var z5731 [1 << 17]byte +var z5732 [1 << 17]byte +var z5733 [1 << 17]byte +var z5734 [1 << 17]byte +var z5735 [1 << 17]byte +var z5736 [1 << 17]byte +var z5737 [1 << 17]byte +var z5738 [1 << 17]byte +var z5739 [1 << 17]byte +var z5740 [1 << 17]byte +var z5741 [1 << 17]byte +var z5742 [1 << 17]byte +var z5743 [1 << 17]byte +var z5744 [1 << 17]byte +var z5745 [1 << 17]byte +var z5746 [1 << 17]byte +var z5747 [1 << 17]byte +var z5748 [1 << 17]byte +var z5749 [1 << 17]byte +var z5750 [1 << 17]byte +var z5751 [1 << 17]byte +var z5752 [1 << 17]byte +var z5753 [1 << 17]byte +var z5754 [1 << 17]byte +var z5755 [1 << 17]byte +var z5756 [1 << 17]byte +var z5757 [1 << 17]byte +var z5758 [1 << 17]byte +var z5759 [1 << 17]byte +var z5760 [1 << 17]byte +var z5761 [1 << 17]byte +var z5762 [1 << 17]byte +var z5763 [1 << 17]byte +var z5764 [1 << 17]byte +var z5765 [1 << 17]byte +var z5766 [1 << 17]byte +var z5767 [1 << 17]byte +var z5768 [1 << 17]byte +var z5769 [1 << 17]byte +var z5770 [1 << 17]byte +var z5771 [1 << 17]byte +var z5772 [1 << 17]byte +var z5773 [1 << 17]byte +var z5774 [1 << 17]byte +var z5775 [1 << 17]byte +var z5776 [1 << 17]byte +var z5777 [1 << 17]byte +var z5778 [1 << 17]byte +var z5779 [1 << 17]byte +var z5780 [1 << 17]byte +var z5781 [1 << 17]byte +var z5782 [1 << 17]byte +var z5783 [1 << 17]byte +var z5784 [1 << 17]byte +var z5785 [1 << 17]byte +var z5786 [1 << 17]byte +var z5787 [1 << 17]byte +var z5788 [1 << 17]byte +var z5789 [1 << 17]byte +var z5790 [1 << 17]byte +var z5791 [1 << 17]byte +var z5792 [1 << 17]byte +var z5793 [1 << 17]byte +var z5794 [1 << 17]byte +var z5795 [1 << 17]byte +var z5796 [1 << 17]byte +var z5797 [1 << 17]byte +var z5798 [1 << 17]byte +var z5799 [1 << 17]byte +var z5800 [1 << 17]byte +var z5801 [1 << 17]byte +var z5802 [1 << 17]byte +var z5803 [1 << 17]byte +var z5804 [1 << 17]byte +var z5805 [1 << 17]byte +var z5806 [1 << 17]byte +var z5807 [1 << 17]byte +var z5808 [1 << 17]byte +var z5809 [1 << 17]byte +var z5810 [1 << 17]byte +var z5811 [1 << 17]byte +var z5812 [1 << 17]byte +var z5813 [1 << 17]byte +var z5814 [1 << 17]byte +var z5815 [1 << 17]byte +var z5816 [1 << 17]byte +var z5817 [1 << 17]byte +var z5818 [1 << 17]byte +var z5819 [1 << 17]byte +var z5820 [1 << 17]byte +var z5821 [1 << 17]byte +var z5822 [1 << 17]byte +var z5823 [1 << 17]byte +var z5824 [1 << 17]byte +var z5825 [1 << 17]byte +var z5826 [1 << 17]byte +var z5827 [1 << 17]byte +var z5828 [1 << 17]byte +var z5829 [1 << 17]byte +var z5830 [1 << 17]byte +var z5831 [1 << 17]byte +var z5832 [1 << 17]byte +var z5833 [1 << 17]byte +var z5834 [1 << 17]byte +var z5835 [1 << 17]byte +var z5836 [1 << 17]byte +var z5837 [1 << 17]byte +var z5838 [1 << 17]byte +var z5839 [1 << 17]byte +var z5840 [1 << 17]byte +var z5841 [1 << 17]byte +var z5842 [1 << 17]byte +var z5843 [1 << 17]byte +var z5844 [1 << 17]byte +var z5845 [1 << 17]byte +var z5846 [1 << 17]byte +var z5847 [1 << 17]byte +var z5848 [1 << 17]byte +var z5849 [1 << 17]byte +var z5850 [1 << 17]byte +var z5851 [1 << 17]byte +var z5852 [1 << 17]byte +var z5853 [1 << 17]byte +var z5854 [1 << 17]byte +var z5855 [1 << 17]byte +var z5856 [1 << 17]byte +var z5857 [1 << 17]byte +var z5858 [1 << 17]byte +var z5859 [1 << 17]byte +var z5860 [1 << 17]byte +var z5861 [1 << 17]byte +var z5862 [1 << 17]byte +var z5863 [1 << 17]byte +var z5864 [1 << 17]byte +var z5865 [1 << 17]byte +var z5866 [1 << 17]byte +var z5867 [1 << 17]byte +var z5868 [1 << 17]byte +var z5869 [1 << 17]byte +var z5870 [1 << 17]byte +var z5871 [1 << 17]byte +var z5872 [1 << 17]byte +var z5873 [1 << 17]byte +var z5874 [1 << 17]byte +var z5875 [1 << 17]byte +var z5876 [1 << 17]byte +var z5877 [1 << 17]byte +var z5878 [1 << 17]byte +var z5879 [1 << 17]byte +var z5880 [1 << 17]byte +var z5881 [1 << 17]byte +var z5882 [1 << 17]byte +var z5883 [1 << 17]byte +var z5884 [1 << 17]byte +var z5885 [1 << 17]byte +var z5886 [1 << 17]byte +var z5887 [1 << 17]byte +var z5888 [1 << 17]byte +var z5889 [1 << 17]byte +var z5890 [1 << 17]byte +var z5891 [1 << 17]byte +var z5892 [1 << 17]byte +var z5893 [1 << 17]byte +var z5894 [1 << 17]byte +var z5895 [1 << 17]byte +var z5896 [1 << 17]byte +var z5897 [1 << 17]byte +var z5898 [1 << 17]byte +var z5899 [1 << 17]byte +var z5900 [1 << 17]byte +var z5901 [1 << 17]byte +var z5902 [1 << 17]byte +var z5903 [1 << 17]byte +var z5904 [1 << 17]byte +var z5905 [1 << 17]byte +var z5906 [1 << 17]byte +var z5907 [1 << 17]byte +var z5908 [1 << 17]byte +var z5909 [1 << 17]byte +var z5910 [1 << 17]byte +var z5911 [1 << 17]byte +var z5912 [1 << 17]byte +var z5913 [1 << 17]byte +var z5914 [1 << 17]byte +var z5915 [1 << 17]byte +var z5916 [1 << 17]byte +var z5917 [1 << 17]byte +var z5918 [1 << 17]byte +var z5919 [1 << 17]byte +var z5920 [1 << 17]byte +var z5921 [1 << 17]byte +var z5922 [1 << 17]byte +var z5923 [1 << 17]byte +var z5924 [1 << 17]byte +var z5925 [1 << 17]byte +var z5926 [1 << 17]byte +var z5927 [1 << 17]byte +var z5928 [1 << 17]byte +var z5929 [1 << 17]byte +var z5930 [1 << 17]byte +var z5931 [1 << 17]byte +var z5932 [1 << 17]byte +var z5933 [1 << 17]byte +var z5934 [1 << 17]byte +var z5935 [1 << 17]byte +var z5936 [1 << 17]byte +var z5937 [1 << 17]byte +var z5938 [1 << 17]byte +var z5939 [1 << 17]byte +var z5940 [1 << 17]byte +var z5941 [1 << 17]byte +var z5942 [1 << 17]byte +var z5943 [1 << 17]byte +var z5944 [1 << 17]byte +var z5945 [1 << 17]byte +var z5946 [1 << 17]byte +var z5947 [1 << 17]byte +var z5948 [1 << 17]byte +var z5949 [1 << 17]byte +var z5950 [1 << 17]byte +var z5951 [1 << 17]byte +var z5952 [1 << 17]byte +var z5953 [1 << 17]byte +var z5954 [1 << 17]byte +var z5955 [1 << 17]byte +var z5956 [1 << 17]byte +var z5957 [1 << 17]byte +var z5958 [1 << 17]byte +var z5959 [1 << 17]byte +var z5960 [1 << 17]byte +var z5961 [1 << 17]byte +var z5962 [1 << 17]byte +var z5963 [1 << 17]byte +var z5964 [1 << 17]byte +var z5965 [1 << 17]byte +var z5966 [1 << 17]byte +var z5967 [1 << 17]byte +var z5968 [1 << 17]byte +var z5969 [1 << 17]byte +var z5970 [1 << 17]byte +var z5971 [1 << 17]byte +var z5972 [1 << 17]byte +var z5973 [1 << 17]byte +var z5974 [1 << 17]byte +var z5975 [1 << 17]byte +var z5976 [1 << 17]byte +var z5977 [1 << 17]byte +var z5978 [1 << 17]byte +var z5979 [1 << 17]byte +var z5980 [1 << 17]byte +var z5981 [1 << 17]byte +var z5982 [1 << 17]byte +var z5983 [1 << 17]byte +var z5984 [1 << 17]byte +var z5985 [1 << 17]byte +var z5986 [1 << 17]byte +var z5987 [1 << 17]byte +var z5988 [1 << 17]byte +var z5989 [1 << 17]byte +var z5990 [1 << 17]byte +var z5991 [1 << 17]byte +var z5992 [1 << 17]byte +var z5993 [1 << 17]byte +var z5994 [1 << 17]byte +var z5995 [1 << 17]byte +var z5996 [1 << 17]byte +var z5997 [1 << 17]byte +var z5998 [1 << 17]byte +var z5999 [1 << 17]byte +var z6000 [1 << 17]byte +var z6001 [1 << 17]byte +var z6002 [1 << 17]byte +var z6003 [1 << 17]byte +var z6004 [1 << 17]byte +var z6005 [1 << 17]byte +var z6006 [1 << 17]byte +var z6007 [1 << 17]byte +var z6008 [1 << 17]byte +var z6009 [1 << 17]byte +var z6010 [1 << 17]byte +var z6011 [1 << 17]byte +var z6012 [1 << 17]byte +var z6013 [1 << 17]byte +var z6014 [1 << 17]byte +var z6015 [1 << 17]byte +var z6016 [1 << 17]byte +var z6017 [1 << 17]byte +var z6018 [1 << 17]byte +var z6019 [1 << 17]byte +var z6020 [1 << 17]byte +var z6021 [1 << 17]byte +var z6022 [1 << 17]byte +var z6023 [1 << 17]byte +var z6024 [1 << 17]byte +var z6025 [1 << 17]byte +var z6026 [1 << 17]byte +var z6027 [1 << 17]byte +var z6028 [1 << 17]byte +var z6029 [1 << 17]byte +var z6030 [1 << 17]byte +var z6031 [1 << 17]byte +var z6032 [1 << 17]byte +var z6033 [1 << 17]byte +var z6034 [1 << 17]byte +var z6035 [1 << 17]byte +var z6036 [1 << 17]byte +var z6037 [1 << 17]byte +var z6038 [1 << 17]byte +var z6039 [1 << 17]byte +var z6040 [1 << 17]byte +var z6041 [1 << 17]byte +var z6042 [1 << 17]byte +var z6043 [1 << 17]byte +var z6044 [1 << 17]byte +var z6045 [1 << 17]byte +var z6046 [1 << 17]byte +var z6047 [1 << 17]byte +var z6048 [1 << 17]byte +var z6049 [1 << 17]byte +var z6050 [1 << 17]byte +var z6051 [1 << 17]byte +var z6052 [1 << 17]byte +var z6053 [1 << 17]byte +var z6054 [1 << 17]byte +var z6055 [1 << 17]byte +var z6056 [1 << 17]byte +var z6057 [1 << 17]byte +var z6058 [1 << 17]byte +var z6059 [1 << 17]byte +var z6060 [1 << 17]byte +var z6061 [1 << 17]byte +var z6062 [1 << 17]byte +var z6063 [1 << 17]byte +var z6064 [1 << 17]byte +var z6065 [1 << 17]byte +var z6066 [1 << 17]byte +var z6067 [1 << 17]byte +var z6068 [1 << 17]byte +var z6069 [1 << 17]byte +var z6070 [1 << 17]byte +var z6071 [1 << 17]byte +var z6072 [1 << 17]byte +var z6073 [1 << 17]byte +var z6074 [1 << 17]byte +var z6075 [1 << 17]byte +var z6076 [1 << 17]byte +var z6077 [1 << 17]byte +var z6078 [1 << 17]byte +var z6079 [1 << 17]byte +var z6080 [1 << 17]byte +var z6081 [1 << 17]byte +var z6082 [1 << 17]byte +var z6083 [1 << 17]byte +var z6084 [1 << 17]byte +var z6085 [1 << 17]byte +var z6086 [1 << 17]byte +var z6087 [1 << 17]byte +var z6088 [1 << 17]byte +var z6089 [1 << 17]byte +var z6090 [1 << 17]byte +var z6091 [1 << 17]byte +var z6092 [1 << 17]byte +var z6093 [1 << 17]byte +var z6094 [1 << 17]byte +var z6095 [1 << 17]byte +var z6096 [1 << 17]byte +var z6097 [1 << 17]byte +var z6098 [1 << 17]byte +var z6099 [1 << 17]byte +var z6100 [1 << 17]byte +var z6101 [1 << 17]byte +var z6102 [1 << 17]byte +var z6103 [1 << 17]byte +var z6104 [1 << 17]byte +var z6105 [1 << 17]byte +var z6106 [1 << 17]byte +var z6107 [1 << 17]byte +var z6108 [1 << 17]byte +var z6109 [1 << 17]byte +var z6110 [1 << 17]byte +var z6111 [1 << 17]byte +var z6112 [1 << 17]byte +var z6113 [1 << 17]byte +var z6114 [1 << 17]byte +var z6115 [1 << 17]byte +var z6116 [1 << 17]byte +var z6117 [1 << 17]byte +var z6118 [1 << 17]byte +var z6119 [1 << 17]byte +var z6120 [1 << 17]byte +var z6121 [1 << 17]byte +var z6122 [1 << 17]byte +var z6123 [1 << 17]byte +var z6124 [1 << 17]byte +var z6125 [1 << 17]byte +var z6126 [1 << 17]byte +var z6127 [1 << 17]byte +var z6128 [1 << 17]byte +var z6129 [1 << 17]byte +var z6130 [1 << 17]byte +var z6131 [1 << 17]byte +var z6132 [1 << 17]byte +var z6133 [1 << 17]byte +var z6134 [1 << 17]byte +var z6135 [1 << 17]byte +var z6136 [1 << 17]byte +var z6137 [1 << 17]byte +var z6138 [1 << 17]byte +var z6139 [1 << 17]byte +var z6140 [1 << 17]byte +var z6141 [1 << 17]byte +var z6142 [1 << 17]byte +var z6143 [1 << 17]byte +var z6144 [1 << 17]byte +var z6145 [1 << 17]byte +var z6146 [1 << 17]byte +var z6147 [1 << 17]byte +var z6148 [1 << 17]byte +var z6149 [1 << 17]byte +var z6150 [1 << 17]byte +var z6151 [1 << 17]byte +var z6152 [1 << 17]byte +var z6153 [1 << 17]byte +var z6154 [1 << 17]byte +var z6155 [1 << 17]byte +var z6156 [1 << 17]byte +var z6157 [1 << 17]byte +var z6158 [1 << 17]byte +var z6159 [1 << 17]byte +var z6160 [1 << 17]byte +var z6161 [1 << 17]byte +var z6162 [1 << 17]byte +var z6163 [1 << 17]byte +var z6164 [1 << 17]byte +var z6165 [1 << 17]byte +var z6166 [1 << 17]byte +var z6167 [1 << 17]byte +var z6168 [1 << 17]byte +var z6169 [1 << 17]byte +var z6170 [1 << 17]byte +var z6171 [1 << 17]byte +var z6172 [1 << 17]byte +var z6173 [1 << 17]byte +var z6174 [1 << 17]byte +var z6175 [1 << 17]byte +var z6176 [1 << 17]byte +var z6177 [1 << 17]byte +var z6178 [1 << 17]byte +var z6179 [1 << 17]byte +var z6180 [1 << 17]byte +var z6181 [1 << 17]byte +var z6182 [1 << 17]byte +var z6183 [1 << 17]byte +var z6184 [1 << 17]byte +var z6185 [1 << 17]byte +var z6186 [1 << 17]byte +var z6187 [1 << 17]byte +var z6188 [1 << 17]byte +var z6189 [1 << 17]byte +var z6190 [1 << 17]byte +var z6191 [1 << 17]byte +var z6192 [1 << 17]byte +var z6193 [1 << 17]byte +var z6194 [1 << 17]byte +var z6195 [1 << 17]byte +var z6196 [1 << 17]byte +var z6197 [1 << 17]byte +var z6198 [1 << 17]byte +var z6199 [1 << 17]byte +var z6200 [1 << 17]byte +var z6201 [1 << 17]byte +var z6202 [1 << 17]byte +var z6203 [1 << 17]byte +var z6204 [1 << 17]byte +var z6205 [1 << 17]byte +var z6206 [1 << 17]byte +var z6207 [1 << 17]byte +var z6208 [1 << 17]byte +var z6209 [1 << 17]byte +var z6210 [1 << 17]byte +var z6211 [1 << 17]byte +var z6212 [1 << 17]byte +var z6213 [1 << 17]byte +var z6214 [1 << 17]byte +var z6215 [1 << 17]byte +var z6216 [1 << 17]byte +var z6217 [1 << 17]byte +var z6218 [1 << 17]byte +var z6219 [1 << 17]byte +var z6220 [1 << 17]byte +var z6221 [1 << 17]byte +var z6222 [1 << 17]byte +var z6223 [1 << 17]byte +var z6224 [1 << 17]byte +var z6225 [1 << 17]byte +var z6226 [1 << 17]byte +var z6227 [1 << 17]byte +var z6228 [1 << 17]byte +var z6229 [1 << 17]byte +var z6230 [1 << 17]byte +var z6231 [1 << 17]byte +var z6232 [1 << 17]byte +var z6233 [1 << 17]byte +var z6234 [1 << 17]byte +var z6235 [1 << 17]byte +var z6236 [1 << 17]byte +var z6237 [1 << 17]byte +var z6238 [1 << 17]byte +var z6239 [1 << 17]byte +var z6240 [1 << 17]byte +var z6241 [1 << 17]byte +var z6242 [1 << 17]byte +var z6243 [1 << 17]byte +var z6244 [1 << 17]byte +var z6245 [1 << 17]byte +var z6246 [1 << 17]byte +var z6247 [1 << 17]byte +var z6248 [1 << 17]byte +var z6249 [1 << 17]byte +var z6250 [1 << 17]byte +var z6251 [1 << 17]byte +var z6252 [1 << 17]byte +var z6253 [1 << 17]byte +var z6254 [1 << 17]byte +var z6255 [1 << 17]byte +var z6256 [1 << 17]byte +var z6257 [1 << 17]byte +var z6258 [1 << 17]byte +var z6259 [1 << 17]byte +var z6260 [1 << 17]byte +var z6261 [1 << 17]byte +var z6262 [1 << 17]byte +var z6263 [1 << 17]byte +var z6264 [1 << 17]byte +var z6265 [1 << 17]byte +var z6266 [1 << 17]byte +var z6267 [1 << 17]byte +var z6268 [1 << 17]byte +var z6269 [1 << 17]byte +var z6270 [1 << 17]byte +var z6271 [1 << 17]byte +var z6272 [1 << 17]byte +var z6273 [1 << 17]byte +var z6274 [1 << 17]byte +var z6275 [1 << 17]byte +var z6276 [1 << 17]byte +var z6277 [1 << 17]byte +var z6278 [1 << 17]byte +var z6279 [1 << 17]byte +var z6280 [1 << 17]byte +var z6281 [1 << 17]byte +var z6282 [1 << 17]byte +var z6283 [1 << 17]byte +var z6284 [1 << 17]byte +var z6285 [1 << 17]byte +var z6286 [1 << 17]byte +var z6287 [1 << 17]byte +var z6288 [1 << 17]byte +var z6289 [1 << 17]byte +var z6290 [1 << 17]byte +var z6291 [1 << 17]byte +var z6292 [1 << 17]byte +var z6293 [1 << 17]byte +var z6294 [1 << 17]byte +var z6295 [1 << 17]byte +var z6296 [1 << 17]byte +var z6297 [1 << 17]byte +var z6298 [1 << 17]byte +var z6299 [1 << 17]byte +var z6300 [1 << 17]byte +var z6301 [1 << 17]byte +var z6302 [1 << 17]byte +var z6303 [1 << 17]byte +var z6304 [1 << 17]byte +var z6305 [1 << 17]byte +var z6306 [1 << 17]byte +var z6307 [1 << 17]byte +var z6308 [1 << 17]byte +var z6309 [1 << 17]byte +var z6310 [1 << 17]byte +var z6311 [1 << 17]byte +var z6312 [1 << 17]byte +var z6313 [1 << 17]byte +var z6314 [1 << 17]byte +var z6315 [1 << 17]byte +var z6316 [1 << 17]byte +var z6317 [1 << 17]byte +var z6318 [1 << 17]byte +var z6319 [1 << 17]byte +var z6320 [1 << 17]byte +var z6321 [1 << 17]byte +var z6322 [1 << 17]byte +var z6323 [1 << 17]byte +var z6324 [1 << 17]byte +var z6325 [1 << 17]byte +var z6326 [1 << 17]byte +var z6327 [1 << 17]byte +var z6328 [1 << 17]byte +var z6329 [1 << 17]byte +var z6330 [1 << 17]byte +var z6331 [1 << 17]byte +var z6332 [1 << 17]byte +var z6333 [1 << 17]byte +var z6334 [1 << 17]byte +var z6335 [1 << 17]byte +var z6336 [1 << 17]byte +var z6337 [1 << 17]byte +var z6338 [1 << 17]byte +var z6339 [1 << 17]byte +var z6340 [1 << 17]byte +var z6341 [1 << 17]byte +var z6342 [1 << 17]byte +var z6343 [1 << 17]byte +var z6344 [1 << 17]byte +var z6345 [1 << 17]byte +var z6346 [1 << 17]byte +var z6347 [1 << 17]byte +var z6348 [1 << 17]byte +var z6349 [1 << 17]byte +var z6350 [1 << 17]byte +var z6351 [1 << 17]byte +var z6352 [1 << 17]byte +var z6353 [1 << 17]byte +var z6354 [1 << 17]byte +var z6355 [1 << 17]byte +var z6356 [1 << 17]byte +var z6357 [1 << 17]byte +var z6358 [1 << 17]byte +var z6359 [1 << 17]byte +var z6360 [1 << 17]byte +var z6361 [1 << 17]byte +var z6362 [1 << 17]byte +var z6363 [1 << 17]byte +var z6364 [1 << 17]byte +var z6365 [1 << 17]byte +var z6366 [1 << 17]byte +var z6367 [1 << 17]byte +var z6368 [1 << 17]byte +var z6369 [1 << 17]byte +var z6370 [1 << 17]byte +var z6371 [1 << 17]byte +var z6372 [1 << 17]byte +var z6373 [1 << 17]byte +var z6374 [1 << 17]byte +var z6375 [1 << 17]byte +var z6376 [1 << 17]byte +var z6377 [1 << 17]byte +var z6378 [1 << 17]byte +var z6379 [1 << 17]byte +var z6380 [1 << 17]byte +var z6381 [1 << 17]byte +var z6382 [1 << 17]byte +var z6383 [1 << 17]byte +var z6384 [1 << 17]byte +var z6385 [1 << 17]byte +var z6386 [1 << 17]byte +var z6387 [1 << 17]byte +var z6388 [1 << 17]byte +var z6389 [1 << 17]byte +var z6390 [1 << 17]byte +var z6391 [1 << 17]byte +var z6392 [1 << 17]byte +var z6393 [1 << 17]byte +var z6394 [1 << 17]byte +var z6395 [1 << 17]byte +var z6396 [1 << 17]byte +var z6397 [1 << 17]byte +var z6398 [1 << 17]byte +var z6399 [1 << 17]byte +var z6400 [1 << 17]byte +var z6401 [1 << 17]byte +var z6402 [1 << 17]byte +var z6403 [1 << 17]byte +var z6404 [1 << 17]byte +var z6405 [1 << 17]byte +var z6406 [1 << 17]byte +var z6407 [1 << 17]byte +var z6408 [1 << 17]byte +var z6409 [1 << 17]byte +var z6410 [1 << 17]byte +var z6411 [1 << 17]byte +var z6412 [1 << 17]byte +var z6413 [1 << 17]byte +var z6414 [1 << 17]byte +var z6415 [1 << 17]byte +var z6416 [1 << 17]byte +var z6417 [1 << 17]byte +var z6418 [1 << 17]byte +var z6419 [1 << 17]byte +var z6420 [1 << 17]byte +var z6421 [1 << 17]byte +var z6422 [1 << 17]byte +var z6423 [1 << 17]byte +var z6424 [1 << 17]byte +var z6425 [1 << 17]byte +var z6426 [1 << 17]byte +var z6427 [1 << 17]byte +var z6428 [1 << 17]byte +var z6429 [1 << 17]byte +var z6430 [1 << 17]byte +var z6431 [1 << 17]byte +var z6432 [1 << 17]byte +var z6433 [1 << 17]byte +var z6434 [1 << 17]byte +var z6435 [1 << 17]byte +var z6436 [1 << 17]byte +var z6437 [1 << 17]byte +var z6438 [1 << 17]byte +var z6439 [1 << 17]byte +var z6440 [1 << 17]byte +var z6441 [1 << 17]byte +var z6442 [1 << 17]byte +var z6443 [1 << 17]byte +var z6444 [1 << 17]byte +var z6445 [1 << 17]byte +var z6446 [1 << 17]byte +var z6447 [1 << 17]byte +var z6448 [1 << 17]byte +var z6449 [1 << 17]byte +var z6450 [1 << 17]byte +var z6451 [1 << 17]byte +var z6452 [1 << 17]byte +var z6453 [1 << 17]byte +var z6454 [1 << 17]byte +var z6455 [1 << 17]byte +var z6456 [1 << 17]byte +var z6457 [1 << 17]byte +var z6458 [1 << 17]byte +var z6459 [1 << 17]byte +var z6460 [1 << 17]byte +var z6461 [1 << 17]byte +var z6462 [1 << 17]byte +var z6463 [1 << 17]byte +var z6464 [1 << 17]byte +var z6465 [1 << 17]byte +var z6466 [1 << 17]byte +var z6467 [1 << 17]byte +var z6468 [1 << 17]byte +var z6469 [1 << 17]byte +var z6470 [1 << 17]byte +var z6471 [1 << 17]byte +var z6472 [1 << 17]byte +var z6473 [1 << 17]byte +var z6474 [1 << 17]byte +var z6475 [1 << 17]byte +var z6476 [1 << 17]byte +var z6477 [1 << 17]byte +var z6478 [1 << 17]byte +var z6479 [1 << 17]byte +var z6480 [1 << 17]byte +var z6481 [1 << 17]byte +var z6482 [1 << 17]byte +var z6483 [1 << 17]byte +var z6484 [1 << 17]byte +var z6485 [1 << 17]byte +var z6486 [1 << 17]byte +var z6487 [1 << 17]byte +var z6488 [1 << 17]byte +var z6489 [1 << 17]byte +var z6490 [1 << 17]byte +var z6491 [1 << 17]byte +var z6492 [1 << 17]byte +var z6493 [1 << 17]byte +var z6494 [1 << 17]byte +var z6495 [1 << 17]byte +var z6496 [1 << 17]byte +var z6497 [1 << 17]byte +var z6498 [1 << 17]byte +var z6499 [1 << 17]byte +var z6500 [1 << 17]byte +var z6501 [1 << 17]byte +var z6502 [1 << 17]byte +var z6503 [1 << 17]byte +var z6504 [1 << 17]byte +var z6505 [1 << 17]byte +var z6506 [1 << 17]byte +var z6507 [1 << 17]byte +var z6508 [1 << 17]byte +var z6509 [1 << 17]byte +var z6510 [1 << 17]byte +var z6511 [1 << 17]byte +var z6512 [1 << 17]byte +var z6513 [1 << 17]byte +var z6514 [1 << 17]byte +var z6515 [1 << 17]byte +var z6516 [1 << 17]byte +var z6517 [1 << 17]byte +var z6518 [1 << 17]byte +var z6519 [1 << 17]byte +var z6520 [1 << 17]byte +var z6521 [1 << 17]byte +var z6522 [1 << 17]byte +var z6523 [1 << 17]byte +var z6524 [1 << 17]byte +var z6525 [1 << 17]byte +var z6526 [1 << 17]byte +var z6527 [1 << 17]byte +var z6528 [1 << 17]byte +var z6529 [1 << 17]byte +var z6530 [1 << 17]byte +var z6531 [1 << 17]byte +var z6532 [1 << 17]byte +var z6533 [1 << 17]byte +var z6534 [1 << 17]byte +var z6535 [1 << 17]byte +var z6536 [1 << 17]byte +var z6537 [1 << 17]byte +var z6538 [1 << 17]byte +var z6539 [1 << 17]byte +var z6540 [1 << 17]byte +var z6541 [1 << 17]byte +var z6542 [1 << 17]byte +var z6543 [1 << 17]byte +var z6544 [1 << 17]byte +var z6545 [1 << 17]byte +var z6546 [1 << 17]byte +var z6547 [1 << 17]byte +var z6548 [1 << 17]byte +var z6549 [1 << 17]byte +var z6550 [1 << 17]byte +var z6551 [1 << 17]byte +var z6552 [1 << 17]byte +var z6553 [1 << 17]byte +var z6554 [1 << 17]byte +var z6555 [1 << 17]byte +var z6556 [1 << 17]byte +var z6557 [1 << 17]byte +var z6558 [1 << 17]byte +var z6559 [1 << 17]byte +var z6560 [1 << 17]byte +var z6561 [1 << 17]byte +var z6562 [1 << 17]byte +var z6563 [1 << 17]byte +var z6564 [1 << 17]byte +var z6565 [1 << 17]byte +var z6566 [1 << 17]byte +var z6567 [1 << 17]byte +var z6568 [1 << 17]byte +var z6569 [1 << 17]byte +var z6570 [1 << 17]byte +var z6571 [1 << 17]byte +var z6572 [1 << 17]byte +var z6573 [1 << 17]byte +var z6574 [1 << 17]byte +var z6575 [1 << 17]byte +var z6576 [1 << 17]byte +var z6577 [1 << 17]byte +var z6578 [1 << 17]byte +var z6579 [1 << 17]byte +var z6580 [1 << 17]byte +var z6581 [1 << 17]byte +var z6582 [1 << 17]byte +var z6583 [1 << 17]byte +var z6584 [1 << 17]byte +var z6585 [1 << 17]byte +var z6586 [1 << 17]byte +var z6587 [1 << 17]byte +var z6588 [1 << 17]byte +var z6589 [1 << 17]byte +var z6590 [1 << 17]byte +var z6591 [1 << 17]byte +var z6592 [1 << 17]byte +var z6593 [1 << 17]byte +var z6594 [1 << 17]byte +var z6595 [1 << 17]byte +var z6596 [1 << 17]byte +var z6597 [1 << 17]byte +var z6598 [1 << 17]byte +var z6599 [1 << 17]byte +var z6600 [1 << 17]byte +var z6601 [1 << 17]byte +var z6602 [1 << 17]byte +var z6603 [1 << 17]byte +var z6604 [1 << 17]byte +var z6605 [1 << 17]byte +var z6606 [1 << 17]byte +var z6607 [1 << 17]byte +var z6608 [1 << 17]byte +var z6609 [1 << 17]byte +var z6610 [1 << 17]byte +var z6611 [1 << 17]byte +var z6612 [1 << 17]byte +var z6613 [1 << 17]byte +var z6614 [1 << 17]byte +var z6615 [1 << 17]byte +var z6616 [1 << 17]byte +var z6617 [1 << 17]byte +var z6618 [1 << 17]byte +var z6619 [1 << 17]byte +var z6620 [1 << 17]byte +var z6621 [1 << 17]byte +var z6622 [1 << 17]byte +var z6623 [1 << 17]byte +var z6624 [1 << 17]byte +var z6625 [1 << 17]byte +var z6626 [1 << 17]byte +var z6627 [1 << 17]byte +var z6628 [1 << 17]byte +var z6629 [1 << 17]byte +var z6630 [1 << 17]byte +var z6631 [1 << 17]byte +var z6632 [1 << 17]byte +var z6633 [1 << 17]byte +var z6634 [1 << 17]byte +var z6635 [1 << 17]byte +var z6636 [1 << 17]byte +var z6637 [1 << 17]byte +var z6638 [1 << 17]byte +var z6639 [1 << 17]byte +var z6640 [1 << 17]byte +var z6641 [1 << 17]byte +var z6642 [1 << 17]byte +var z6643 [1 << 17]byte +var z6644 [1 << 17]byte +var z6645 [1 << 17]byte +var z6646 [1 << 17]byte +var z6647 [1 << 17]byte +var z6648 [1 << 17]byte +var z6649 [1 << 17]byte +var z6650 [1 << 17]byte +var z6651 [1 << 17]byte +var z6652 [1 << 17]byte +var z6653 [1 << 17]byte +var z6654 [1 << 17]byte +var z6655 [1 << 17]byte +var z6656 [1 << 17]byte +var z6657 [1 << 17]byte +var z6658 [1 << 17]byte +var z6659 [1 << 17]byte +var z6660 [1 << 17]byte +var z6661 [1 << 17]byte +var z6662 [1 << 17]byte +var z6663 [1 << 17]byte +var z6664 [1 << 17]byte +var z6665 [1 << 17]byte +var z6666 [1 << 17]byte +var z6667 [1 << 17]byte +var z6668 [1 << 17]byte +var z6669 [1 << 17]byte +var z6670 [1 << 17]byte +var z6671 [1 << 17]byte +var z6672 [1 << 17]byte +var z6673 [1 << 17]byte +var z6674 [1 << 17]byte +var z6675 [1 << 17]byte +var z6676 [1 << 17]byte +var z6677 [1 << 17]byte +var z6678 [1 << 17]byte +var z6679 [1 << 17]byte +var z6680 [1 << 17]byte +var z6681 [1 << 17]byte +var z6682 [1 << 17]byte +var z6683 [1 << 17]byte +var z6684 [1 << 17]byte +var z6685 [1 << 17]byte +var z6686 [1 << 17]byte +var z6687 [1 << 17]byte +var z6688 [1 << 17]byte +var z6689 [1 << 17]byte +var z6690 [1 << 17]byte +var z6691 [1 << 17]byte +var z6692 [1 << 17]byte +var z6693 [1 << 17]byte +var z6694 [1 << 17]byte +var z6695 [1 << 17]byte +var z6696 [1 << 17]byte +var z6697 [1 << 17]byte +var z6698 [1 << 17]byte +var z6699 [1 << 17]byte +var z6700 [1 << 17]byte +var z6701 [1 << 17]byte +var z6702 [1 << 17]byte +var z6703 [1 << 17]byte +var z6704 [1 << 17]byte +var z6705 [1 << 17]byte +var z6706 [1 << 17]byte +var z6707 [1 << 17]byte +var z6708 [1 << 17]byte +var z6709 [1 << 17]byte +var z6710 [1 << 17]byte +var z6711 [1 << 17]byte +var z6712 [1 << 17]byte +var z6713 [1 << 17]byte +var z6714 [1 << 17]byte +var z6715 [1 << 17]byte +var z6716 [1 << 17]byte +var z6717 [1 << 17]byte +var z6718 [1 << 17]byte +var z6719 [1 << 17]byte +var z6720 [1 << 17]byte +var z6721 [1 << 17]byte +var z6722 [1 << 17]byte +var z6723 [1 << 17]byte +var z6724 [1 << 17]byte +var z6725 [1 << 17]byte +var z6726 [1 << 17]byte +var z6727 [1 << 17]byte +var z6728 [1 << 17]byte +var z6729 [1 << 17]byte +var z6730 [1 << 17]byte +var z6731 [1 << 17]byte +var z6732 [1 << 17]byte +var z6733 [1 << 17]byte +var z6734 [1 << 17]byte +var z6735 [1 << 17]byte +var z6736 [1 << 17]byte +var z6737 [1 << 17]byte +var z6738 [1 << 17]byte +var z6739 [1 << 17]byte +var z6740 [1 << 17]byte +var z6741 [1 << 17]byte +var z6742 [1 << 17]byte +var z6743 [1 << 17]byte +var z6744 [1 << 17]byte +var z6745 [1 << 17]byte +var z6746 [1 << 17]byte +var z6747 [1 << 17]byte +var z6748 [1 << 17]byte +var z6749 [1 << 17]byte +var z6750 [1 << 17]byte +var z6751 [1 << 17]byte +var z6752 [1 << 17]byte +var z6753 [1 << 17]byte +var z6754 [1 << 17]byte +var z6755 [1 << 17]byte +var z6756 [1 << 17]byte +var z6757 [1 << 17]byte +var z6758 [1 << 17]byte +var z6759 [1 << 17]byte +var z6760 [1 << 17]byte +var z6761 [1 << 17]byte +var z6762 [1 << 17]byte +var z6763 [1 << 17]byte +var z6764 [1 << 17]byte +var z6765 [1 << 17]byte +var z6766 [1 << 17]byte +var z6767 [1 << 17]byte +var z6768 [1 << 17]byte +var z6769 [1 << 17]byte +var z6770 [1 << 17]byte +var z6771 [1 << 17]byte +var z6772 [1 << 17]byte +var z6773 [1 << 17]byte +var z6774 [1 << 17]byte +var z6775 [1 << 17]byte +var z6776 [1 << 17]byte +var z6777 [1 << 17]byte +var z6778 [1 << 17]byte +var z6779 [1 << 17]byte +var z6780 [1 << 17]byte +var z6781 [1 << 17]byte +var z6782 [1 << 17]byte +var z6783 [1 << 17]byte +var z6784 [1 << 17]byte +var z6785 [1 << 17]byte +var z6786 [1 << 17]byte +var z6787 [1 << 17]byte +var z6788 [1 << 17]byte +var z6789 [1 << 17]byte +var z6790 [1 << 17]byte +var z6791 [1 << 17]byte +var z6792 [1 << 17]byte +var z6793 [1 << 17]byte +var z6794 [1 << 17]byte +var z6795 [1 << 17]byte +var z6796 [1 << 17]byte +var z6797 [1 << 17]byte +var z6798 [1 << 17]byte +var z6799 [1 << 17]byte +var z6800 [1 << 17]byte +var z6801 [1 << 17]byte +var z6802 [1 << 17]byte +var z6803 [1 << 17]byte +var z6804 [1 << 17]byte +var z6805 [1 << 17]byte +var z6806 [1 << 17]byte +var z6807 [1 << 17]byte +var z6808 [1 << 17]byte +var z6809 [1 << 17]byte +var z6810 [1 << 17]byte +var z6811 [1 << 17]byte +var z6812 [1 << 17]byte +var z6813 [1 << 17]byte +var z6814 [1 << 17]byte +var z6815 [1 << 17]byte +var z6816 [1 << 17]byte +var z6817 [1 << 17]byte +var z6818 [1 << 17]byte +var z6819 [1 << 17]byte +var z6820 [1 << 17]byte +var z6821 [1 << 17]byte +var z6822 [1 << 17]byte +var z6823 [1 << 17]byte +var z6824 [1 << 17]byte +var z6825 [1 << 17]byte +var z6826 [1 << 17]byte +var z6827 [1 << 17]byte +var z6828 [1 << 17]byte +var z6829 [1 << 17]byte +var z6830 [1 << 17]byte +var z6831 [1 << 17]byte +var z6832 [1 << 17]byte +var z6833 [1 << 17]byte +var z6834 [1 << 17]byte +var z6835 [1 << 17]byte +var z6836 [1 << 17]byte +var z6837 [1 << 17]byte +var z6838 [1 << 17]byte +var z6839 [1 << 17]byte +var z6840 [1 << 17]byte +var z6841 [1 << 17]byte +var z6842 [1 << 17]byte +var z6843 [1 << 17]byte +var z6844 [1 << 17]byte +var z6845 [1 << 17]byte +var z6846 [1 << 17]byte +var z6847 [1 << 17]byte +var z6848 [1 << 17]byte +var z6849 [1 << 17]byte +var z6850 [1 << 17]byte +var z6851 [1 << 17]byte +var z6852 [1 << 17]byte +var z6853 [1 << 17]byte +var z6854 [1 << 17]byte +var z6855 [1 << 17]byte +var z6856 [1 << 17]byte +var z6857 [1 << 17]byte +var z6858 [1 << 17]byte +var z6859 [1 << 17]byte +var z6860 [1 << 17]byte +var z6861 [1 << 17]byte +var z6862 [1 << 17]byte +var z6863 [1 << 17]byte +var z6864 [1 << 17]byte +var z6865 [1 << 17]byte +var z6866 [1 << 17]byte +var z6867 [1 << 17]byte +var z6868 [1 << 17]byte +var z6869 [1 << 17]byte +var z6870 [1 << 17]byte +var z6871 [1 << 17]byte +var z6872 [1 << 17]byte +var z6873 [1 << 17]byte +var z6874 [1 << 17]byte +var z6875 [1 << 17]byte +var z6876 [1 << 17]byte +var z6877 [1 << 17]byte +var z6878 [1 << 17]byte +var z6879 [1 << 17]byte +var z6880 [1 << 17]byte +var z6881 [1 << 17]byte +var z6882 [1 << 17]byte +var z6883 [1 << 17]byte +var z6884 [1 << 17]byte +var z6885 [1 << 17]byte +var z6886 [1 << 17]byte +var z6887 [1 << 17]byte +var z6888 [1 << 17]byte +var z6889 [1 << 17]byte +var z6890 [1 << 17]byte +var z6891 [1 << 17]byte +var z6892 [1 << 17]byte +var z6893 [1 << 17]byte +var z6894 [1 << 17]byte +var z6895 [1 << 17]byte +var z6896 [1 << 17]byte +var z6897 [1 << 17]byte +var z6898 [1 << 17]byte +var z6899 [1 << 17]byte +var z6900 [1 << 17]byte +var z6901 [1 << 17]byte +var z6902 [1 << 17]byte +var z6903 [1 << 17]byte +var z6904 [1 << 17]byte +var z6905 [1 << 17]byte +var z6906 [1 << 17]byte +var z6907 [1 << 17]byte +var z6908 [1 << 17]byte +var z6909 [1 << 17]byte +var z6910 [1 << 17]byte +var z6911 [1 << 17]byte +var z6912 [1 << 17]byte +var z6913 [1 << 17]byte +var z6914 [1 << 17]byte +var z6915 [1 << 17]byte +var z6916 [1 << 17]byte +var z6917 [1 << 17]byte +var z6918 [1 << 17]byte +var z6919 [1 << 17]byte +var z6920 [1 << 17]byte +var z6921 [1 << 17]byte +var z6922 [1 << 17]byte +var z6923 [1 << 17]byte +var z6924 [1 << 17]byte +var z6925 [1 << 17]byte +var z6926 [1 << 17]byte +var z6927 [1 << 17]byte +var z6928 [1 << 17]byte +var z6929 [1 << 17]byte +var z6930 [1 << 17]byte +var z6931 [1 << 17]byte +var z6932 [1 << 17]byte +var z6933 [1 << 17]byte +var z6934 [1 << 17]byte +var z6935 [1 << 17]byte +var z6936 [1 << 17]byte +var z6937 [1 << 17]byte +var z6938 [1 << 17]byte +var z6939 [1 << 17]byte +var z6940 [1 << 17]byte +var z6941 [1 << 17]byte +var z6942 [1 << 17]byte +var z6943 [1 << 17]byte +var z6944 [1 << 17]byte +var z6945 [1 << 17]byte +var z6946 [1 << 17]byte +var z6947 [1 << 17]byte +var z6948 [1 << 17]byte +var z6949 [1 << 17]byte +var z6950 [1 << 17]byte +var z6951 [1 << 17]byte +var z6952 [1 << 17]byte +var z6953 [1 << 17]byte +var z6954 [1 << 17]byte +var z6955 [1 << 17]byte +var z6956 [1 << 17]byte +var z6957 [1 << 17]byte +var z6958 [1 << 17]byte +var z6959 [1 << 17]byte +var z6960 [1 << 17]byte +var z6961 [1 << 17]byte +var z6962 [1 << 17]byte +var z6963 [1 << 17]byte +var z6964 [1 << 17]byte +var z6965 [1 << 17]byte +var z6966 [1 << 17]byte +var z6967 [1 << 17]byte +var z6968 [1 << 17]byte +var z6969 [1 << 17]byte +var z6970 [1 << 17]byte +var z6971 [1 << 17]byte +var z6972 [1 << 17]byte +var z6973 [1 << 17]byte +var z6974 [1 << 17]byte +var z6975 [1 << 17]byte +var z6976 [1 << 17]byte +var z6977 [1 << 17]byte +var z6978 [1 << 17]byte +var z6979 [1 << 17]byte +var z6980 [1 << 17]byte +var z6981 [1 << 17]byte +var z6982 [1 << 17]byte +var z6983 [1 << 17]byte +var z6984 [1 << 17]byte +var z6985 [1 << 17]byte +var z6986 [1 << 17]byte +var z6987 [1 << 17]byte +var z6988 [1 << 17]byte +var z6989 [1 << 17]byte +var z6990 [1 << 17]byte +var z6991 [1 << 17]byte +var z6992 [1 << 17]byte +var z6993 [1 << 17]byte +var z6994 [1 << 17]byte +var z6995 [1 << 17]byte +var z6996 [1 << 17]byte +var z6997 [1 << 17]byte +var z6998 [1 << 17]byte +var z6999 [1 << 17]byte +var z7000 [1 << 17]byte +var z7001 [1 << 17]byte +var z7002 [1 << 17]byte +var z7003 [1 << 17]byte +var z7004 [1 << 17]byte +var z7005 [1 << 17]byte +var z7006 [1 << 17]byte +var z7007 [1 << 17]byte +var z7008 [1 << 17]byte +var z7009 [1 << 17]byte +var z7010 [1 << 17]byte +var z7011 [1 << 17]byte +var z7012 [1 << 17]byte +var z7013 [1 << 17]byte +var z7014 [1 << 17]byte +var z7015 [1 << 17]byte +var z7016 [1 << 17]byte +var z7017 [1 << 17]byte +var z7018 [1 << 17]byte +var z7019 [1 << 17]byte +var z7020 [1 << 17]byte +var z7021 [1 << 17]byte +var z7022 [1 << 17]byte +var z7023 [1 << 17]byte +var z7024 [1 << 17]byte +var z7025 [1 << 17]byte +var z7026 [1 << 17]byte +var z7027 [1 << 17]byte +var z7028 [1 << 17]byte +var z7029 [1 << 17]byte +var z7030 [1 << 17]byte +var z7031 [1 << 17]byte +var z7032 [1 << 17]byte +var z7033 [1 << 17]byte +var z7034 [1 << 17]byte +var z7035 [1 << 17]byte +var z7036 [1 << 17]byte +var z7037 [1 << 17]byte +var z7038 [1 << 17]byte +var z7039 [1 << 17]byte +var z7040 [1 << 17]byte +var z7041 [1 << 17]byte +var z7042 [1 << 17]byte +var z7043 [1 << 17]byte +var z7044 [1 << 17]byte +var z7045 [1 << 17]byte +var z7046 [1 << 17]byte +var z7047 [1 << 17]byte +var z7048 [1 << 17]byte +var z7049 [1 << 17]byte +var z7050 [1 << 17]byte +var z7051 [1 << 17]byte +var z7052 [1 << 17]byte +var z7053 [1 << 17]byte +var z7054 [1 << 17]byte +var z7055 [1 << 17]byte +var z7056 [1 << 17]byte +var z7057 [1 << 17]byte +var z7058 [1 << 17]byte +var z7059 [1 << 17]byte +var z7060 [1 << 17]byte +var z7061 [1 << 17]byte +var z7062 [1 << 17]byte +var z7063 [1 << 17]byte +var z7064 [1 << 17]byte +var z7065 [1 << 17]byte +var z7066 [1 << 17]byte +var z7067 [1 << 17]byte +var z7068 [1 << 17]byte +var z7069 [1 << 17]byte +var z7070 [1 << 17]byte +var z7071 [1 << 17]byte +var z7072 [1 << 17]byte +var z7073 [1 << 17]byte +var z7074 [1 << 17]byte +var z7075 [1 << 17]byte +var z7076 [1 << 17]byte +var z7077 [1 << 17]byte +var z7078 [1 << 17]byte +var z7079 [1 << 17]byte +var z7080 [1 << 17]byte +var z7081 [1 << 17]byte +var z7082 [1 << 17]byte +var z7083 [1 << 17]byte +var z7084 [1 << 17]byte +var z7085 [1 << 17]byte +var z7086 [1 << 17]byte +var z7087 [1 << 17]byte +var z7088 [1 << 17]byte +var z7089 [1 << 17]byte +var z7090 [1 << 17]byte +var z7091 [1 << 17]byte +var z7092 [1 << 17]byte +var z7093 [1 << 17]byte +var z7094 [1 << 17]byte +var z7095 [1 << 17]byte +var z7096 [1 << 17]byte +var z7097 [1 << 17]byte +var z7098 [1 << 17]byte +var z7099 [1 << 17]byte +var z7100 [1 << 17]byte +var z7101 [1 << 17]byte +var z7102 [1 << 17]byte +var z7103 [1 << 17]byte +var z7104 [1 << 17]byte +var z7105 [1 << 17]byte +var z7106 [1 << 17]byte +var z7107 [1 << 17]byte +var z7108 [1 << 17]byte +var z7109 [1 << 17]byte +var z7110 [1 << 17]byte +var z7111 [1 << 17]byte +var z7112 [1 << 17]byte +var z7113 [1 << 17]byte +var z7114 [1 << 17]byte +var z7115 [1 << 17]byte +var z7116 [1 << 17]byte +var z7117 [1 << 17]byte +var z7118 [1 << 17]byte +var z7119 [1 << 17]byte +var z7120 [1 << 17]byte +var z7121 [1 << 17]byte +var z7122 [1 << 17]byte +var z7123 [1 << 17]byte +var z7124 [1 << 17]byte +var z7125 [1 << 17]byte +var z7126 [1 << 17]byte +var z7127 [1 << 17]byte +var z7128 [1 << 17]byte +var z7129 [1 << 17]byte +var z7130 [1 << 17]byte +var z7131 [1 << 17]byte +var z7132 [1 << 17]byte +var z7133 [1 << 17]byte +var z7134 [1 << 17]byte +var z7135 [1 << 17]byte +var z7136 [1 << 17]byte +var z7137 [1 << 17]byte +var z7138 [1 << 17]byte +var z7139 [1 << 17]byte +var z7140 [1 << 17]byte +var z7141 [1 << 17]byte +var z7142 [1 << 17]byte +var z7143 [1 << 17]byte +var z7144 [1 << 17]byte +var z7145 [1 << 17]byte +var z7146 [1 << 17]byte +var z7147 [1 << 17]byte +var z7148 [1 << 17]byte +var z7149 [1 << 17]byte +var z7150 [1 << 17]byte +var z7151 [1 << 17]byte +var z7152 [1 << 17]byte +var z7153 [1 << 17]byte +var z7154 [1 << 17]byte +var z7155 [1 << 17]byte +var z7156 [1 << 17]byte +var z7157 [1 << 17]byte +var z7158 [1 << 17]byte +var z7159 [1 << 17]byte +var z7160 [1 << 17]byte +var z7161 [1 << 17]byte +var z7162 [1 << 17]byte +var z7163 [1 << 17]byte +var z7164 [1 << 17]byte +var z7165 [1 << 17]byte +var z7166 [1 << 17]byte +var z7167 [1 << 17]byte +var z7168 [1 << 17]byte +var z7169 [1 << 17]byte +var z7170 [1 << 17]byte +var z7171 [1 << 17]byte +var z7172 [1 << 17]byte +var z7173 [1 << 17]byte +var z7174 [1 << 17]byte +var z7175 [1 << 17]byte +var z7176 [1 << 17]byte +var z7177 [1 << 17]byte +var z7178 [1 << 17]byte +var z7179 [1 << 17]byte +var z7180 [1 << 17]byte +var z7181 [1 << 17]byte +var z7182 [1 << 17]byte +var z7183 [1 << 17]byte +var z7184 [1 << 17]byte +var z7185 [1 << 17]byte +var z7186 [1 << 17]byte +var z7187 [1 << 17]byte +var z7188 [1 << 17]byte +var z7189 [1 << 17]byte +var z7190 [1 << 17]byte +var z7191 [1 << 17]byte +var z7192 [1 << 17]byte +var z7193 [1 << 17]byte +var z7194 [1 << 17]byte +var z7195 [1 << 17]byte +var z7196 [1 << 17]byte +var z7197 [1 << 17]byte +var z7198 [1 << 17]byte +var z7199 [1 << 17]byte +var z7200 [1 << 17]byte +var z7201 [1 << 17]byte +var z7202 [1 << 17]byte +var z7203 [1 << 17]byte +var z7204 [1 << 17]byte +var z7205 [1 << 17]byte +var z7206 [1 << 17]byte +var z7207 [1 << 17]byte +var z7208 [1 << 17]byte +var z7209 [1 << 17]byte +var z7210 [1 << 17]byte +var z7211 [1 << 17]byte +var z7212 [1 << 17]byte +var z7213 [1 << 17]byte +var z7214 [1 << 17]byte +var z7215 [1 << 17]byte +var z7216 [1 << 17]byte +var z7217 [1 << 17]byte +var z7218 [1 << 17]byte +var z7219 [1 << 17]byte +var z7220 [1 << 17]byte +var z7221 [1 << 17]byte +var z7222 [1 << 17]byte +var z7223 [1 << 17]byte +var z7224 [1 << 17]byte +var z7225 [1 << 17]byte +var z7226 [1 << 17]byte +var z7227 [1 << 17]byte +var z7228 [1 << 17]byte +var z7229 [1 << 17]byte +var z7230 [1 << 17]byte +var z7231 [1 << 17]byte +var z7232 [1 << 17]byte +var z7233 [1 << 17]byte +var z7234 [1 << 17]byte +var z7235 [1 << 17]byte +var z7236 [1 << 17]byte +var z7237 [1 << 17]byte +var z7238 [1 << 17]byte +var z7239 [1 << 17]byte +var z7240 [1 << 17]byte +var z7241 [1 << 17]byte +var z7242 [1 << 17]byte +var z7243 [1 << 17]byte +var z7244 [1 << 17]byte +var z7245 [1 << 17]byte +var z7246 [1 << 17]byte +var z7247 [1 << 17]byte +var z7248 [1 << 17]byte +var z7249 [1 << 17]byte +var z7250 [1 << 17]byte +var z7251 [1 << 17]byte +var z7252 [1 << 17]byte +var z7253 [1 << 17]byte +var z7254 [1 << 17]byte +var z7255 [1 << 17]byte +var z7256 [1 << 17]byte +var z7257 [1 << 17]byte +var z7258 [1 << 17]byte +var z7259 [1 << 17]byte +var z7260 [1 << 17]byte +var z7261 [1 << 17]byte +var z7262 [1 << 17]byte +var z7263 [1 << 17]byte +var z7264 [1 << 17]byte +var z7265 [1 << 17]byte +var z7266 [1 << 17]byte +var z7267 [1 << 17]byte +var z7268 [1 << 17]byte +var z7269 [1 << 17]byte +var z7270 [1 << 17]byte +var z7271 [1 << 17]byte +var z7272 [1 << 17]byte +var z7273 [1 << 17]byte +var z7274 [1 << 17]byte +var z7275 [1 << 17]byte +var z7276 [1 << 17]byte +var z7277 [1 << 17]byte +var z7278 [1 << 17]byte +var z7279 [1 << 17]byte +var z7280 [1 << 17]byte +var z7281 [1 << 17]byte +var z7282 [1 << 17]byte +var z7283 [1 << 17]byte +var z7284 [1 << 17]byte +var z7285 [1 << 17]byte +var z7286 [1 << 17]byte +var z7287 [1 << 17]byte +var z7288 [1 << 17]byte +var z7289 [1 << 17]byte +var z7290 [1 << 17]byte +var z7291 [1 << 17]byte +var z7292 [1 << 17]byte +var z7293 [1 << 17]byte +var z7294 [1 << 17]byte +var z7295 [1 << 17]byte +var z7296 [1 << 17]byte +var z7297 [1 << 17]byte +var z7298 [1 << 17]byte +var z7299 [1 << 17]byte +var z7300 [1 << 17]byte +var z7301 [1 << 17]byte +var z7302 [1 << 17]byte +var z7303 [1 << 17]byte +var z7304 [1 << 17]byte +var z7305 [1 << 17]byte +var z7306 [1 << 17]byte +var z7307 [1 << 17]byte +var z7308 [1 << 17]byte +var z7309 [1 << 17]byte +var z7310 [1 << 17]byte +var z7311 [1 << 17]byte +var z7312 [1 << 17]byte +var z7313 [1 << 17]byte +var z7314 [1 << 17]byte +var z7315 [1 << 17]byte +var z7316 [1 << 17]byte +var z7317 [1 << 17]byte +var z7318 [1 << 17]byte +var z7319 [1 << 17]byte +var z7320 [1 << 17]byte +var z7321 [1 << 17]byte +var z7322 [1 << 17]byte +var z7323 [1 << 17]byte +var z7324 [1 << 17]byte +var z7325 [1 << 17]byte +var z7326 [1 << 17]byte +var z7327 [1 << 17]byte +var z7328 [1 << 17]byte +var z7329 [1 << 17]byte +var z7330 [1 << 17]byte +var z7331 [1 << 17]byte +var z7332 [1 << 17]byte +var z7333 [1 << 17]byte +var z7334 [1 << 17]byte +var z7335 [1 << 17]byte +var z7336 [1 << 17]byte +var z7337 [1 << 17]byte +var z7338 [1 << 17]byte +var z7339 [1 << 17]byte +var z7340 [1 << 17]byte +var z7341 [1 << 17]byte +var z7342 [1 << 17]byte +var z7343 [1 << 17]byte +var z7344 [1 << 17]byte +var z7345 [1 << 17]byte +var z7346 [1 << 17]byte +var z7347 [1 << 17]byte +var z7348 [1 << 17]byte +var z7349 [1 << 17]byte +var z7350 [1 << 17]byte +var z7351 [1 << 17]byte +var z7352 [1 << 17]byte +var z7353 [1 << 17]byte +var z7354 [1 << 17]byte +var z7355 [1 << 17]byte +var z7356 [1 << 17]byte +var z7357 [1 << 17]byte +var z7358 [1 << 17]byte +var z7359 [1 << 17]byte +var z7360 [1 << 17]byte +var z7361 [1 << 17]byte +var z7362 [1 << 17]byte +var z7363 [1 << 17]byte +var z7364 [1 << 17]byte +var z7365 [1 << 17]byte +var z7366 [1 << 17]byte +var z7367 [1 << 17]byte +var z7368 [1 << 17]byte +var z7369 [1 << 17]byte +var z7370 [1 << 17]byte +var z7371 [1 << 17]byte +var z7372 [1 << 17]byte +var z7373 [1 << 17]byte +var z7374 [1 << 17]byte +var z7375 [1 << 17]byte +var z7376 [1 << 17]byte +var z7377 [1 << 17]byte +var z7378 [1 << 17]byte +var z7379 [1 << 17]byte +var z7380 [1 << 17]byte +var z7381 [1 << 17]byte +var z7382 [1 << 17]byte +var z7383 [1 << 17]byte +var z7384 [1 << 17]byte +var z7385 [1 << 17]byte +var z7386 [1 << 17]byte +var z7387 [1 << 17]byte +var z7388 [1 << 17]byte +var z7389 [1 << 17]byte +var z7390 [1 << 17]byte +var z7391 [1 << 17]byte +var z7392 [1 << 17]byte +var z7393 [1 << 17]byte +var z7394 [1 << 17]byte +var z7395 [1 << 17]byte +var z7396 [1 << 17]byte +var z7397 [1 << 17]byte +var z7398 [1 << 17]byte +var z7399 [1 << 17]byte +var z7400 [1 << 17]byte +var z7401 [1 << 17]byte +var z7402 [1 << 17]byte +var z7403 [1 << 17]byte +var z7404 [1 << 17]byte +var z7405 [1 << 17]byte +var z7406 [1 << 17]byte +var z7407 [1 << 17]byte +var z7408 [1 << 17]byte +var z7409 [1 << 17]byte +var z7410 [1 << 17]byte +var z7411 [1 << 17]byte +var z7412 [1 << 17]byte +var z7413 [1 << 17]byte +var z7414 [1 << 17]byte +var z7415 [1 << 17]byte +var z7416 [1 << 17]byte +var z7417 [1 << 17]byte +var z7418 [1 << 17]byte +var z7419 [1 << 17]byte +var z7420 [1 << 17]byte +var z7421 [1 << 17]byte +var z7422 [1 << 17]byte +var z7423 [1 << 17]byte +var z7424 [1 << 17]byte +var z7425 [1 << 17]byte +var z7426 [1 << 17]byte +var z7427 [1 << 17]byte +var z7428 [1 << 17]byte +var z7429 [1 << 17]byte +var z7430 [1 << 17]byte +var z7431 [1 << 17]byte +var z7432 [1 << 17]byte +var z7433 [1 << 17]byte +var z7434 [1 << 17]byte +var z7435 [1 << 17]byte +var z7436 [1 << 17]byte +var z7437 [1 << 17]byte +var z7438 [1 << 17]byte +var z7439 [1 << 17]byte +var z7440 [1 << 17]byte +var z7441 [1 << 17]byte +var z7442 [1 << 17]byte +var z7443 [1 << 17]byte +var z7444 [1 << 17]byte +var z7445 [1 << 17]byte +var z7446 [1 << 17]byte +var z7447 [1 << 17]byte +var z7448 [1 << 17]byte +var z7449 [1 << 17]byte +var z7450 [1 << 17]byte +var z7451 [1 << 17]byte +var z7452 [1 << 17]byte +var z7453 [1 << 17]byte +var z7454 [1 << 17]byte +var z7455 [1 << 17]byte +var z7456 [1 << 17]byte +var z7457 [1 << 17]byte +var z7458 [1 << 17]byte +var z7459 [1 << 17]byte +var z7460 [1 << 17]byte +var z7461 [1 << 17]byte +var z7462 [1 << 17]byte +var z7463 [1 << 17]byte +var z7464 [1 << 17]byte +var z7465 [1 << 17]byte +var z7466 [1 << 17]byte +var z7467 [1 << 17]byte +var z7468 [1 << 17]byte +var z7469 [1 << 17]byte +var z7470 [1 << 17]byte +var z7471 [1 << 17]byte +var z7472 [1 << 17]byte +var z7473 [1 << 17]byte +var z7474 [1 << 17]byte +var z7475 [1 << 17]byte +var z7476 [1 << 17]byte +var z7477 [1 << 17]byte +var z7478 [1 << 17]byte +var z7479 [1 << 17]byte +var z7480 [1 << 17]byte +var z7481 [1 << 17]byte +var z7482 [1 << 17]byte +var z7483 [1 << 17]byte +var z7484 [1 << 17]byte +var z7485 [1 << 17]byte +var z7486 [1 << 17]byte +var z7487 [1 << 17]byte +var z7488 [1 << 17]byte +var z7489 [1 << 17]byte +var z7490 [1 << 17]byte +var z7491 [1 << 17]byte +var z7492 [1 << 17]byte +var z7493 [1 << 17]byte +var z7494 [1 << 17]byte +var z7495 [1 << 17]byte +var z7496 [1 << 17]byte +var z7497 [1 << 17]byte +var z7498 [1 << 17]byte +var z7499 [1 << 17]byte +var z7500 [1 << 17]byte +var z7501 [1 << 17]byte +var z7502 [1 << 17]byte +var z7503 [1 << 17]byte +var z7504 [1 << 17]byte +var z7505 [1 << 17]byte +var z7506 [1 << 17]byte +var z7507 [1 << 17]byte +var z7508 [1 << 17]byte +var z7509 [1 << 17]byte +var z7510 [1 << 17]byte +var z7511 [1 << 17]byte +var z7512 [1 << 17]byte +var z7513 [1 << 17]byte +var z7514 [1 << 17]byte +var z7515 [1 << 17]byte +var z7516 [1 << 17]byte +var z7517 [1 << 17]byte +var z7518 [1 << 17]byte +var z7519 [1 << 17]byte +var z7520 [1 << 17]byte +var z7521 [1 << 17]byte +var z7522 [1 << 17]byte +var z7523 [1 << 17]byte +var z7524 [1 << 17]byte +var z7525 [1 << 17]byte +var z7526 [1 << 17]byte +var z7527 [1 << 17]byte +var z7528 [1 << 17]byte +var z7529 [1 << 17]byte +var z7530 [1 << 17]byte +var z7531 [1 << 17]byte +var z7532 [1 << 17]byte +var z7533 [1 << 17]byte +var z7534 [1 << 17]byte +var z7535 [1 << 17]byte +var z7536 [1 << 17]byte +var z7537 [1 << 17]byte +var z7538 [1 << 17]byte +var z7539 [1 << 17]byte +var z7540 [1 << 17]byte +var z7541 [1 << 17]byte +var z7542 [1 << 17]byte +var z7543 [1 << 17]byte +var z7544 [1 << 17]byte +var z7545 [1 << 17]byte +var z7546 [1 << 17]byte +var z7547 [1 << 17]byte +var z7548 [1 << 17]byte +var z7549 [1 << 17]byte +var z7550 [1 << 17]byte +var z7551 [1 << 17]byte +var z7552 [1 << 17]byte +var z7553 [1 << 17]byte +var z7554 [1 << 17]byte +var z7555 [1 << 17]byte +var z7556 [1 << 17]byte +var z7557 [1 << 17]byte +var z7558 [1 << 17]byte +var z7559 [1 << 17]byte +var z7560 [1 << 17]byte +var z7561 [1 << 17]byte +var z7562 [1 << 17]byte +var z7563 [1 << 17]byte +var z7564 [1 << 17]byte +var z7565 [1 << 17]byte +var z7566 [1 << 17]byte +var z7567 [1 << 17]byte +var z7568 [1 << 17]byte +var z7569 [1 << 17]byte +var z7570 [1 << 17]byte +var z7571 [1 << 17]byte +var z7572 [1 << 17]byte +var z7573 [1 << 17]byte +var z7574 [1 << 17]byte +var z7575 [1 << 17]byte +var z7576 [1 << 17]byte +var z7577 [1 << 17]byte +var z7578 [1 << 17]byte +var z7579 [1 << 17]byte +var z7580 [1 << 17]byte +var z7581 [1 << 17]byte +var z7582 [1 << 17]byte +var z7583 [1 << 17]byte +var z7584 [1 << 17]byte +var z7585 [1 << 17]byte +var z7586 [1 << 17]byte +var z7587 [1 << 17]byte +var z7588 [1 << 17]byte +var z7589 [1 << 17]byte +var z7590 [1 << 17]byte +var z7591 [1 << 17]byte +var z7592 [1 << 17]byte +var z7593 [1 << 17]byte +var z7594 [1 << 17]byte +var z7595 [1 << 17]byte +var z7596 [1 << 17]byte +var z7597 [1 << 17]byte +var z7598 [1 << 17]byte +var z7599 [1 << 17]byte +var z7600 [1 << 17]byte +var z7601 [1 << 17]byte +var z7602 [1 << 17]byte +var z7603 [1 << 17]byte +var z7604 [1 << 17]byte +var z7605 [1 << 17]byte +var z7606 [1 << 17]byte +var z7607 [1 << 17]byte +var z7608 [1 << 17]byte +var z7609 [1 << 17]byte +var z7610 [1 << 17]byte +var z7611 [1 << 17]byte +var z7612 [1 << 17]byte +var z7613 [1 << 17]byte +var z7614 [1 << 17]byte +var z7615 [1 << 17]byte +var z7616 [1 << 17]byte +var z7617 [1 << 17]byte +var z7618 [1 << 17]byte +var z7619 [1 << 17]byte +var z7620 [1 << 17]byte +var z7621 [1 << 17]byte +var z7622 [1 << 17]byte +var z7623 [1 << 17]byte +var z7624 [1 << 17]byte +var z7625 [1 << 17]byte +var z7626 [1 << 17]byte +var z7627 [1 << 17]byte +var z7628 [1 << 17]byte +var z7629 [1 << 17]byte +var z7630 [1 << 17]byte +var z7631 [1 << 17]byte +var z7632 [1 << 17]byte +var z7633 [1 << 17]byte +var z7634 [1 << 17]byte +var z7635 [1 << 17]byte +var z7636 [1 << 17]byte +var z7637 [1 << 17]byte +var z7638 [1 << 17]byte +var z7639 [1 << 17]byte +var z7640 [1 << 17]byte +var z7641 [1 << 17]byte +var z7642 [1 << 17]byte +var z7643 [1 << 17]byte +var z7644 [1 << 17]byte +var z7645 [1 << 17]byte +var z7646 [1 << 17]byte +var z7647 [1 << 17]byte +var z7648 [1 << 17]byte +var z7649 [1 << 17]byte +var z7650 [1 << 17]byte +var z7651 [1 << 17]byte +var z7652 [1 << 17]byte +var z7653 [1 << 17]byte +var z7654 [1 << 17]byte +var z7655 [1 << 17]byte +var z7656 [1 << 17]byte +var z7657 [1 << 17]byte +var z7658 [1 << 17]byte +var z7659 [1 << 17]byte +var z7660 [1 << 17]byte +var z7661 [1 << 17]byte +var z7662 [1 << 17]byte +var z7663 [1 << 17]byte +var z7664 [1 << 17]byte +var z7665 [1 << 17]byte +var z7666 [1 << 17]byte +var z7667 [1 << 17]byte +var z7668 [1 << 17]byte +var z7669 [1 << 17]byte +var z7670 [1 << 17]byte +var z7671 [1 << 17]byte +var z7672 [1 << 17]byte +var z7673 [1 << 17]byte +var z7674 [1 << 17]byte +var z7675 [1 << 17]byte +var z7676 [1 << 17]byte +var z7677 [1 << 17]byte +var z7678 [1 << 17]byte +var z7679 [1 << 17]byte +var z7680 [1 << 17]byte +var z7681 [1 << 17]byte +var z7682 [1 << 17]byte +var z7683 [1 << 17]byte +var z7684 [1 << 17]byte +var z7685 [1 << 17]byte +var z7686 [1 << 17]byte +var z7687 [1 << 17]byte +var z7688 [1 << 17]byte +var z7689 [1 << 17]byte +var z7690 [1 << 17]byte +var z7691 [1 << 17]byte +var z7692 [1 << 17]byte +var z7693 [1 << 17]byte +var z7694 [1 << 17]byte +var z7695 [1 << 17]byte +var z7696 [1 << 17]byte +var z7697 [1 << 17]byte +var z7698 [1 << 17]byte +var z7699 [1 << 17]byte +var z7700 [1 << 17]byte +var z7701 [1 << 17]byte +var z7702 [1 << 17]byte +var z7703 [1 << 17]byte +var z7704 [1 << 17]byte +var z7705 [1 << 17]byte +var z7706 [1 << 17]byte +var z7707 [1 << 17]byte +var z7708 [1 << 17]byte +var z7709 [1 << 17]byte +var z7710 [1 << 17]byte +var z7711 [1 << 17]byte +var z7712 [1 << 17]byte +var z7713 [1 << 17]byte +var z7714 [1 << 17]byte +var z7715 [1 << 17]byte +var z7716 [1 << 17]byte +var z7717 [1 << 17]byte +var z7718 [1 << 17]byte +var z7719 [1 << 17]byte +var z7720 [1 << 17]byte +var z7721 [1 << 17]byte +var z7722 [1 << 17]byte +var z7723 [1 << 17]byte +var z7724 [1 << 17]byte +var z7725 [1 << 17]byte +var z7726 [1 << 17]byte +var z7727 [1 << 17]byte +var z7728 [1 << 17]byte +var z7729 [1 << 17]byte +var z7730 [1 << 17]byte +var z7731 [1 << 17]byte +var z7732 [1 << 17]byte +var z7733 [1 << 17]byte +var z7734 [1 << 17]byte +var z7735 [1 << 17]byte +var z7736 [1 << 17]byte +var z7737 [1 << 17]byte +var z7738 [1 << 17]byte +var z7739 [1 << 17]byte +var z7740 [1 << 17]byte +var z7741 [1 << 17]byte +var z7742 [1 << 17]byte +var z7743 [1 << 17]byte +var z7744 [1 << 17]byte +var z7745 [1 << 17]byte +var z7746 [1 << 17]byte +var z7747 [1 << 17]byte +var z7748 [1 << 17]byte +var z7749 [1 << 17]byte +var z7750 [1 << 17]byte +var z7751 [1 << 17]byte +var z7752 [1 << 17]byte +var z7753 [1 << 17]byte +var z7754 [1 << 17]byte +var z7755 [1 << 17]byte +var z7756 [1 << 17]byte +var z7757 [1 << 17]byte +var z7758 [1 << 17]byte +var z7759 [1 << 17]byte +var z7760 [1 << 17]byte +var z7761 [1 << 17]byte +var z7762 [1 << 17]byte +var z7763 [1 << 17]byte +var z7764 [1 << 17]byte +var z7765 [1 << 17]byte +var z7766 [1 << 17]byte +var z7767 [1 << 17]byte +var z7768 [1 << 17]byte +var z7769 [1 << 17]byte +var z7770 [1 << 17]byte +var z7771 [1 << 17]byte +var z7772 [1 << 17]byte +var z7773 [1 << 17]byte +var z7774 [1 << 17]byte +var z7775 [1 << 17]byte +var z7776 [1 << 17]byte +var z7777 [1 << 17]byte +var z7778 [1 << 17]byte +var z7779 [1 << 17]byte +var z7780 [1 << 17]byte +var z7781 [1 << 17]byte +var z7782 [1 << 17]byte +var z7783 [1 << 17]byte +var z7784 [1 << 17]byte +var z7785 [1 << 17]byte +var z7786 [1 << 17]byte +var z7787 [1 << 17]byte +var z7788 [1 << 17]byte +var z7789 [1 << 17]byte +var z7790 [1 << 17]byte +var z7791 [1 << 17]byte +var z7792 [1 << 17]byte +var z7793 [1 << 17]byte +var z7794 [1 << 17]byte +var z7795 [1 << 17]byte +var z7796 [1 << 17]byte +var z7797 [1 << 17]byte +var z7798 [1 << 17]byte +var z7799 [1 << 17]byte +var z7800 [1 << 17]byte +var z7801 [1 << 17]byte +var z7802 [1 << 17]byte +var z7803 [1 << 17]byte +var z7804 [1 << 17]byte +var z7805 [1 << 17]byte +var z7806 [1 << 17]byte +var z7807 [1 << 17]byte +var z7808 [1 << 17]byte +var z7809 [1 << 17]byte +var z7810 [1 << 17]byte +var z7811 [1 << 17]byte +var z7812 [1 << 17]byte +var z7813 [1 << 17]byte +var z7814 [1 << 17]byte +var z7815 [1 << 17]byte +var z7816 [1 << 17]byte +var z7817 [1 << 17]byte +var z7818 [1 << 17]byte +var z7819 [1 << 17]byte +var z7820 [1 << 17]byte +var z7821 [1 << 17]byte +var z7822 [1 << 17]byte +var z7823 [1 << 17]byte +var z7824 [1 << 17]byte +var z7825 [1 << 17]byte +var z7826 [1 << 17]byte +var z7827 [1 << 17]byte +var z7828 [1 << 17]byte +var z7829 [1 << 17]byte +var z7830 [1 << 17]byte +var z7831 [1 << 17]byte +var z7832 [1 << 17]byte +var z7833 [1 << 17]byte +var z7834 [1 << 17]byte +var z7835 [1 << 17]byte +var z7836 [1 << 17]byte +var z7837 [1 << 17]byte +var z7838 [1 << 17]byte +var z7839 [1 << 17]byte +var z7840 [1 << 17]byte +var z7841 [1 << 17]byte +var z7842 [1 << 17]byte +var z7843 [1 << 17]byte +var z7844 [1 << 17]byte +var z7845 [1 << 17]byte +var z7846 [1 << 17]byte +var z7847 [1 << 17]byte +var z7848 [1 << 17]byte +var z7849 [1 << 17]byte +var z7850 [1 << 17]byte +var z7851 [1 << 17]byte +var z7852 [1 << 17]byte +var z7853 [1 << 17]byte +var z7854 [1 << 17]byte +var z7855 [1 << 17]byte +var z7856 [1 << 17]byte +var z7857 [1 << 17]byte +var z7858 [1 << 17]byte +var z7859 [1 << 17]byte +var z7860 [1 << 17]byte +var z7861 [1 << 17]byte +var z7862 [1 << 17]byte +var z7863 [1 << 17]byte +var z7864 [1 << 17]byte +var z7865 [1 << 17]byte +var z7866 [1 << 17]byte +var z7867 [1 << 17]byte +var z7868 [1 << 17]byte +var z7869 [1 << 17]byte +var z7870 [1 << 17]byte +var z7871 [1 << 17]byte +var z7872 [1 << 17]byte +var z7873 [1 << 17]byte +var z7874 [1 << 17]byte +var z7875 [1 << 17]byte +var z7876 [1 << 17]byte +var z7877 [1 << 17]byte +var z7878 [1 << 17]byte +var z7879 [1 << 17]byte +var z7880 [1 << 17]byte +var z7881 [1 << 17]byte +var z7882 [1 << 17]byte +var z7883 [1 << 17]byte +var z7884 [1 << 17]byte +var z7885 [1 << 17]byte +var z7886 [1 << 17]byte +var z7887 [1 << 17]byte +var z7888 [1 << 17]byte +var z7889 [1 << 17]byte +var z7890 [1 << 17]byte +var z7891 [1 << 17]byte +var z7892 [1 << 17]byte +var z7893 [1 << 17]byte +var z7894 [1 << 17]byte +var z7895 [1 << 17]byte +var z7896 [1 << 17]byte +var z7897 [1 << 17]byte +var z7898 [1 << 17]byte +var z7899 [1 << 17]byte +var z7900 [1 << 17]byte +var z7901 [1 << 17]byte +var z7902 [1 << 17]byte +var z7903 [1 << 17]byte +var z7904 [1 << 17]byte +var z7905 [1 << 17]byte +var z7906 [1 << 17]byte +var z7907 [1 << 17]byte +var z7908 [1 << 17]byte +var z7909 [1 << 17]byte +var z7910 [1 << 17]byte +var z7911 [1 << 17]byte +var z7912 [1 << 17]byte +var z7913 [1 << 17]byte +var z7914 [1 << 17]byte +var z7915 [1 << 17]byte +var z7916 [1 << 17]byte +var z7917 [1 << 17]byte +var z7918 [1 << 17]byte +var z7919 [1 << 17]byte +var z7920 [1 << 17]byte +var z7921 [1 << 17]byte +var z7922 [1 << 17]byte +var z7923 [1 << 17]byte +var z7924 [1 << 17]byte +var z7925 [1 << 17]byte +var z7926 [1 << 17]byte +var z7927 [1 << 17]byte +var z7928 [1 << 17]byte +var z7929 [1 << 17]byte +var z7930 [1 << 17]byte +var z7931 [1 << 17]byte +var z7932 [1 << 17]byte +var z7933 [1 << 17]byte +var z7934 [1 << 17]byte +var z7935 [1 << 17]byte +var z7936 [1 << 17]byte +var z7937 [1 << 17]byte +var z7938 [1 << 17]byte +var z7939 [1 << 17]byte +var z7940 [1 << 17]byte +var z7941 [1 << 17]byte +var z7942 [1 << 17]byte +var z7943 [1 << 17]byte +var z7944 [1 << 17]byte +var z7945 [1 << 17]byte +var z7946 [1 << 17]byte +var z7947 [1 << 17]byte +var z7948 [1 << 17]byte +var z7949 [1 << 17]byte +var z7950 [1 << 17]byte +var z7951 [1 << 17]byte +var z7952 [1 << 17]byte +var z7953 [1 << 17]byte +var z7954 [1 << 17]byte +var z7955 [1 << 17]byte +var z7956 [1 << 17]byte +var z7957 [1 << 17]byte +var z7958 [1 << 17]byte +var z7959 [1 << 17]byte +var z7960 [1 << 17]byte +var z7961 [1 << 17]byte +var z7962 [1 << 17]byte +var z7963 [1 << 17]byte +var z7964 [1 << 17]byte +var z7965 [1 << 17]byte +var z7966 [1 << 17]byte +var z7967 [1 << 17]byte +var z7968 [1 << 17]byte +var z7969 [1 << 17]byte +var z7970 [1 << 17]byte +var z7971 [1 << 17]byte +var z7972 [1 << 17]byte +var z7973 [1 << 17]byte +var z7974 [1 << 17]byte +var z7975 [1 << 17]byte +var z7976 [1 << 17]byte +var z7977 [1 << 17]byte +var z7978 [1 << 17]byte +var z7979 [1 << 17]byte +var z7980 [1 << 17]byte +var z7981 [1 << 17]byte +var z7982 [1 << 17]byte +var z7983 [1 << 17]byte +var z7984 [1 << 17]byte +var z7985 [1 << 17]byte +var z7986 [1 << 17]byte +var z7987 [1 << 17]byte +var z7988 [1 << 17]byte +var z7989 [1 << 17]byte +var z7990 [1 << 17]byte +var z7991 [1 << 17]byte +var z7992 [1 << 17]byte +var z7993 [1 << 17]byte +var z7994 [1 << 17]byte +var z7995 [1 << 17]byte +var z7996 [1 << 17]byte +var z7997 [1 << 17]byte +var z7998 [1 << 17]byte +var z7999 [1 << 17]byte +var z8000 [1 << 17]byte +var z8001 [1 << 17]byte +var z8002 [1 << 17]byte +var z8003 [1 << 17]byte +var z8004 [1 << 17]byte +var z8005 [1 << 17]byte +var z8006 [1 << 17]byte +var z8007 [1 << 17]byte +var z8008 [1 << 17]byte +var z8009 [1 << 17]byte +var z8010 [1 << 17]byte +var z8011 [1 << 17]byte +var z8012 [1 << 17]byte +var z8013 [1 << 17]byte +var z8014 [1 << 17]byte +var z8015 [1 << 17]byte +var z8016 [1 << 17]byte +var z8017 [1 << 17]byte +var z8018 [1 << 17]byte +var z8019 [1 << 17]byte +var z8020 [1 << 17]byte +var z8021 [1 << 17]byte +var z8022 [1 << 17]byte +var z8023 [1 << 17]byte +var z8024 [1 << 17]byte +var z8025 [1 << 17]byte +var z8026 [1 << 17]byte +var z8027 [1 << 17]byte +var z8028 [1 << 17]byte +var z8029 [1 << 17]byte +var z8030 [1 << 17]byte +var z8031 [1 << 17]byte +var z8032 [1 << 17]byte +var z8033 [1 << 17]byte +var z8034 [1 << 17]byte +var z8035 [1 << 17]byte +var z8036 [1 << 17]byte +var z8037 [1 << 17]byte +var z8038 [1 << 17]byte +var z8039 [1 << 17]byte +var z8040 [1 << 17]byte +var z8041 [1 << 17]byte +var z8042 [1 << 17]byte +var z8043 [1 << 17]byte +var z8044 [1 << 17]byte +var z8045 [1 << 17]byte +var z8046 [1 << 17]byte +var z8047 [1 << 17]byte +var z8048 [1 << 17]byte +var z8049 [1 << 17]byte +var z8050 [1 << 17]byte +var z8051 [1 << 17]byte +var z8052 [1 << 17]byte +var z8053 [1 << 17]byte +var z8054 [1 << 17]byte +var z8055 [1 << 17]byte +var z8056 [1 << 17]byte +var z8057 [1 << 17]byte +var z8058 [1 << 17]byte +var z8059 [1 << 17]byte +var z8060 [1 << 17]byte +var z8061 [1 << 17]byte +var z8062 [1 << 17]byte +var z8063 [1 << 17]byte +var z8064 [1 << 17]byte +var z8065 [1 << 17]byte +var z8066 [1 << 17]byte +var z8067 [1 << 17]byte +var z8068 [1 << 17]byte +var z8069 [1 << 17]byte +var z8070 [1 << 17]byte +var z8071 [1 << 17]byte +var z8072 [1 << 17]byte +var z8073 [1 << 17]byte +var z8074 [1 << 17]byte +var z8075 [1 << 17]byte +var z8076 [1 << 17]byte +var z8077 [1 << 17]byte +var z8078 [1 << 17]byte +var z8079 [1 << 17]byte +var z8080 [1 << 17]byte +var z8081 [1 << 17]byte +var z8082 [1 << 17]byte +var z8083 [1 << 17]byte +var z8084 [1 << 17]byte +var z8085 [1 << 17]byte +var z8086 [1 << 17]byte +var z8087 [1 << 17]byte +var z8088 [1 << 17]byte +var z8089 [1 << 17]byte +var z8090 [1 << 17]byte +var z8091 [1 << 17]byte +var z8092 [1 << 17]byte +var z8093 [1 << 17]byte +var z8094 [1 << 17]byte +var z8095 [1 << 17]byte +var z8096 [1 << 17]byte +var z8097 [1 << 17]byte +var z8098 [1 << 17]byte +var z8099 [1 << 17]byte +var z8100 [1 << 17]byte +var z8101 [1 << 17]byte +var z8102 [1 << 17]byte +var z8103 [1 << 17]byte +var z8104 [1 << 17]byte +var z8105 [1 << 17]byte +var z8106 [1 << 17]byte +var z8107 [1 << 17]byte +var z8108 [1 << 17]byte +var z8109 [1 << 17]byte +var z8110 [1 << 17]byte +var z8111 [1 << 17]byte +var z8112 [1 << 17]byte +var z8113 [1 << 17]byte +var z8114 [1 << 17]byte +var z8115 [1 << 17]byte +var z8116 [1 << 17]byte +var z8117 [1 << 17]byte +var z8118 [1 << 17]byte +var z8119 [1 << 17]byte +var z8120 [1 << 17]byte +var z8121 [1 << 17]byte +var z8122 [1 << 17]byte +var z8123 [1 << 17]byte +var z8124 [1 << 17]byte +var z8125 [1 << 17]byte +var z8126 [1 << 17]byte +var z8127 [1 << 17]byte +var z8128 [1 << 17]byte +var z8129 [1 << 17]byte +var z8130 [1 << 17]byte +var z8131 [1 << 17]byte +var z8132 [1 << 17]byte +var z8133 [1 << 17]byte +var z8134 [1 << 17]byte +var z8135 [1 << 17]byte +var z8136 [1 << 17]byte +var z8137 [1 << 17]byte +var z8138 [1 << 17]byte +var z8139 [1 << 17]byte +var z8140 [1 << 17]byte +var z8141 [1 << 17]byte +var z8142 [1 << 17]byte +var z8143 [1 << 17]byte +var z8144 [1 << 17]byte +var z8145 [1 << 17]byte +var z8146 [1 << 17]byte +var z8147 [1 << 17]byte +var z8148 [1 << 17]byte +var z8149 [1 << 17]byte +var z8150 [1 << 17]byte +var z8151 [1 << 17]byte +var z8152 [1 << 17]byte +var z8153 [1 << 17]byte +var z8154 [1 << 17]byte +var z8155 [1 << 17]byte +var z8156 [1 << 17]byte +var z8157 [1 << 17]byte +var z8158 [1 << 17]byte +var z8159 [1 << 17]byte +var z8160 [1 << 17]byte +var z8161 [1 << 17]byte +var z8162 [1 << 17]byte +var z8163 [1 << 17]byte +var z8164 [1 << 17]byte +var z8165 [1 << 17]byte +var z8166 [1 << 17]byte +var z8167 [1 << 17]byte +var z8168 [1 << 17]byte +var z8169 [1 << 17]byte +var z8170 [1 << 17]byte +var z8171 [1 << 17]byte +var z8172 [1 << 17]byte +var z8173 [1 << 17]byte +var z8174 [1 << 17]byte +var z8175 [1 << 17]byte +var z8176 [1 << 17]byte +var z8177 [1 << 17]byte +var z8178 [1 << 17]byte +var z8179 [1 << 17]byte +var z8180 [1 << 17]byte +var z8181 [1 << 17]byte +var z8182 [1 << 17]byte +var z8183 [1 << 17]byte +var z8184 [1 << 17]byte +var z8185 [1 << 17]byte +var z8186 [1 << 17]byte +var z8187 [1 << 17]byte +var z8188 [1 << 17]byte +var z8189 [1 << 17]byte +var z8190 [1 << 17]byte +var z8191 [1 << 17]byte +var z8192 [1 << 17]byte +var z8193 [1 << 17]byte +var z8194 [1 << 17]byte +var z8195 [1 << 17]byte +var z8196 [1 << 17]byte +var z8197 [1 << 17]byte +var z8198 [1 << 17]byte +var z8199 [1 << 17]byte +var z8200 [1 << 17]byte +var z8201 [1 << 17]byte +var z8202 [1 << 17]byte +var z8203 [1 << 17]byte +var z8204 [1 << 17]byte +var z8205 [1 << 17]byte +var z8206 [1 << 17]byte +var z8207 [1 << 17]byte +var z8208 [1 << 17]byte +var z8209 [1 << 17]byte +var z8210 [1 << 17]byte +var z8211 [1 << 17]byte +var z8212 [1 << 17]byte +var z8213 [1 << 17]byte +var z8214 [1 << 17]byte +var z8215 [1 << 17]byte +var z8216 [1 << 17]byte +var z8217 [1 << 17]byte +var z8218 [1 << 17]byte +var z8219 [1 << 17]byte +var z8220 [1 << 17]byte +var z8221 [1 << 17]byte +var z8222 [1 << 17]byte +var z8223 [1 << 17]byte +var z8224 [1 << 17]byte +var z8225 [1 << 17]byte +var z8226 [1 << 17]byte +var z8227 [1 << 17]byte +var z8228 [1 << 17]byte +var z8229 [1 << 17]byte +var z8230 [1 << 17]byte +var z8231 [1 << 17]byte +var z8232 [1 << 17]byte +var z8233 [1 << 17]byte +var z8234 [1 << 17]byte +var z8235 [1 << 17]byte +var z8236 [1 << 17]byte +var z8237 [1 << 17]byte +var z8238 [1 << 17]byte +var z8239 [1 << 17]byte +var z8240 [1 << 17]byte +var z8241 [1 << 17]byte +var z8242 [1 << 17]byte +var z8243 [1 << 17]byte +var z8244 [1 << 17]byte +var z8245 [1 << 17]byte +var z8246 [1 << 17]byte +var z8247 [1 << 17]byte +var z8248 [1 << 17]byte +var z8249 [1 << 17]byte +var z8250 [1 << 17]byte +var z8251 [1 << 17]byte +var z8252 [1 << 17]byte +var z8253 [1 << 17]byte +var z8254 [1 << 17]byte +var z8255 [1 << 17]byte +var z8256 [1 << 17]byte +var z8257 [1 << 17]byte +var z8258 [1 << 17]byte +var z8259 [1 << 17]byte +var z8260 [1 << 17]byte +var z8261 [1 << 17]byte +var z8262 [1 << 17]byte +var z8263 [1 << 17]byte +var z8264 [1 << 17]byte +var z8265 [1 << 17]byte +var z8266 [1 << 17]byte +var z8267 [1 << 17]byte +var z8268 [1 << 17]byte +var z8269 [1 << 17]byte +var z8270 [1 << 17]byte +var z8271 [1 << 17]byte +var z8272 [1 << 17]byte +var z8273 [1 << 17]byte +var z8274 [1 << 17]byte +var z8275 [1 << 17]byte +var z8276 [1 << 17]byte +var z8277 [1 << 17]byte +var z8278 [1 << 17]byte +var z8279 [1 << 17]byte +var z8280 [1 << 17]byte +var z8281 [1 << 17]byte +var z8282 [1 << 17]byte +var z8283 [1 << 17]byte +var z8284 [1 << 17]byte +var z8285 [1 << 17]byte +var z8286 [1 << 17]byte +var z8287 [1 << 17]byte +var z8288 [1 << 17]byte +var z8289 [1 << 17]byte +var z8290 [1 << 17]byte +var z8291 [1 << 17]byte +var z8292 [1 << 17]byte +var z8293 [1 << 17]byte +var z8294 [1 << 17]byte +var z8295 [1 << 17]byte +var z8296 [1 << 17]byte +var z8297 [1 << 17]byte +var z8298 [1 << 17]byte +var z8299 [1 << 17]byte +var z8300 [1 << 17]byte +var z8301 [1 << 17]byte +var z8302 [1 << 17]byte +var z8303 [1 << 17]byte +var z8304 [1 << 17]byte +var z8305 [1 << 17]byte +var z8306 [1 << 17]byte +var z8307 [1 << 17]byte +var z8308 [1 << 17]byte +var z8309 [1 << 17]byte +var z8310 [1 << 17]byte +var z8311 [1 << 17]byte +var z8312 [1 << 17]byte +var z8313 [1 << 17]byte +var z8314 [1 << 17]byte +var z8315 [1 << 17]byte +var z8316 [1 << 17]byte +var z8317 [1 << 17]byte +var z8318 [1 << 17]byte +var z8319 [1 << 17]byte +var z8320 [1 << 17]byte +var z8321 [1 << 17]byte +var z8322 [1 << 17]byte +var z8323 [1 << 17]byte +var z8324 [1 << 17]byte +var z8325 [1 << 17]byte +var z8326 [1 << 17]byte +var z8327 [1 << 17]byte +var z8328 [1 << 17]byte +var z8329 [1 << 17]byte +var z8330 [1 << 17]byte +var z8331 [1 << 17]byte +var z8332 [1 << 17]byte +var z8333 [1 << 17]byte +var z8334 [1 << 17]byte +var z8335 [1 << 17]byte +var z8336 [1 << 17]byte +var z8337 [1 << 17]byte +var z8338 [1 << 17]byte +var z8339 [1 << 17]byte +var z8340 [1 << 17]byte +var z8341 [1 << 17]byte +var z8342 [1 << 17]byte +var z8343 [1 << 17]byte +var z8344 [1 << 17]byte +var z8345 [1 << 17]byte +var z8346 [1 << 17]byte +var z8347 [1 << 17]byte +var z8348 [1 << 17]byte +var z8349 [1 << 17]byte +var z8350 [1 << 17]byte +var z8351 [1 << 17]byte +var z8352 [1 << 17]byte +var z8353 [1 << 17]byte +var z8354 [1 << 17]byte +var z8355 [1 << 17]byte +var z8356 [1 << 17]byte +var z8357 [1 << 17]byte +var z8358 [1 << 17]byte +var z8359 [1 << 17]byte +var z8360 [1 << 17]byte +var z8361 [1 << 17]byte +var z8362 [1 << 17]byte +var z8363 [1 << 17]byte +var z8364 [1 << 17]byte +var z8365 [1 << 17]byte +var z8366 [1 << 17]byte +var z8367 [1 << 17]byte +var z8368 [1 << 17]byte +var z8369 [1 << 17]byte +var z8370 [1 << 17]byte +var z8371 [1 << 17]byte +var z8372 [1 << 17]byte +var z8373 [1 << 17]byte +var z8374 [1 << 17]byte +var z8375 [1 << 17]byte +var z8376 [1 << 17]byte +var z8377 [1 << 17]byte +var z8378 [1 << 17]byte +var z8379 [1 << 17]byte +var z8380 [1 << 17]byte +var z8381 [1 << 17]byte +var z8382 [1 << 17]byte +var z8383 [1 << 17]byte +var z8384 [1 << 17]byte +var z8385 [1 << 17]byte +var z8386 [1 << 17]byte +var z8387 [1 << 17]byte +var z8388 [1 << 17]byte +var z8389 [1 << 17]byte +var z8390 [1 << 17]byte +var z8391 [1 << 17]byte +var z8392 [1 << 17]byte +var z8393 [1 << 17]byte +var z8394 [1 << 17]byte +var z8395 [1 << 17]byte +var z8396 [1 << 17]byte +var z8397 [1 << 17]byte +var z8398 [1 << 17]byte +var z8399 [1 << 17]byte +var z8400 [1 << 17]byte +var z8401 [1 << 17]byte +var z8402 [1 << 17]byte +var z8403 [1 << 17]byte +var z8404 [1 << 17]byte +var z8405 [1 << 17]byte +var z8406 [1 << 17]byte +var z8407 [1 << 17]byte +var z8408 [1 << 17]byte +var z8409 [1 << 17]byte +var z8410 [1 << 17]byte +var z8411 [1 << 17]byte +var z8412 [1 << 17]byte +var z8413 [1 << 17]byte +var z8414 [1 << 17]byte +var z8415 [1 << 17]byte +var z8416 [1 << 17]byte +var z8417 [1 << 17]byte +var z8418 [1 << 17]byte +var z8419 [1 << 17]byte +var z8420 [1 << 17]byte +var z8421 [1 << 17]byte +var z8422 [1 << 17]byte +var z8423 [1 << 17]byte +var z8424 [1 << 17]byte +var z8425 [1 << 17]byte +var z8426 [1 << 17]byte +var z8427 [1 << 17]byte +var z8428 [1 << 17]byte +var z8429 [1 << 17]byte +var z8430 [1 << 17]byte +var z8431 [1 << 17]byte +var z8432 [1 << 17]byte +var z8433 [1 << 17]byte +var z8434 [1 << 17]byte +var z8435 [1 << 17]byte +var z8436 [1 << 17]byte +var z8437 [1 << 17]byte +var z8438 [1 << 17]byte +var z8439 [1 << 17]byte +var z8440 [1 << 17]byte +var z8441 [1 << 17]byte +var z8442 [1 << 17]byte +var z8443 [1 << 17]byte +var z8444 [1 << 17]byte +var z8445 [1 << 17]byte +var z8446 [1 << 17]byte +var z8447 [1 << 17]byte +var z8448 [1 << 17]byte +var z8449 [1 << 17]byte +var z8450 [1 << 17]byte +var z8451 [1 << 17]byte +var z8452 [1 << 17]byte +var z8453 [1 << 17]byte +var z8454 [1 << 17]byte +var z8455 [1 << 17]byte +var z8456 [1 << 17]byte +var z8457 [1 << 17]byte +var z8458 [1 << 17]byte +var z8459 [1 << 17]byte +var z8460 [1 << 17]byte +var z8461 [1 << 17]byte +var z8462 [1 << 17]byte +var z8463 [1 << 17]byte +var z8464 [1 << 17]byte +var z8465 [1 << 17]byte +var z8466 [1 << 17]byte +var z8467 [1 << 17]byte +var z8468 [1 << 17]byte +var z8469 [1 << 17]byte +var z8470 [1 << 17]byte +var z8471 [1 << 17]byte +var z8472 [1 << 17]byte +var z8473 [1 << 17]byte +var z8474 [1 << 17]byte +var z8475 [1 << 17]byte +var z8476 [1 << 17]byte +var z8477 [1 << 17]byte +var z8478 [1 << 17]byte +var z8479 [1 << 17]byte +var z8480 [1 << 17]byte +var z8481 [1 << 17]byte +var z8482 [1 << 17]byte +var z8483 [1 << 17]byte +var z8484 [1 << 17]byte +var z8485 [1 << 17]byte +var z8486 [1 << 17]byte +var z8487 [1 << 17]byte +var z8488 [1 << 17]byte +var z8489 [1 << 17]byte +var z8490 [1 << 17]byte +var z8491 [1 << 17]byte +var z8492 [1 << 17]byte +var z8493 [1 << 17]byte +var z8494 [1 << 17]byte +var z8495 [1 << 17]byte +var z8496 [1 << 17]byte +var z8497 [1 << 17]byte +var z8498 [1 << 17]byte +var z8499 [1 << 17]byte +var z8500 [1 << 17]byte +var z8501 [1 << 17]byte +var z8502 [1 << 17]byte +var z8503 [1 << 17]byte +var z8504 [1 << 17]byte +var z8505 [1 << 17]byte +var z8506 [1 << 17]byte +var z8507 [1 << 17]byte +var z8508 [1 << 17]byte +var z8509 [1 << 17]byte +var z8510 [1 << 17]byte +var z8511 [1 << 17]byte +var z8512 [1 << 17]byte +var z8513 [1 << 17]byte +var z8514 [1 << 17]byte +var z8515 [1 << 17]byte +var z8516 [1 << 17]byte +var z8517 [1 << 17]byte +var z8518 [1 << 17]byte +var z8519 [1 << 17]byte +var z8520 [1 << 17]byte +var z8521 [1 << 17]byte +var z8522 [1 << 17]byte +var z8523 [1 << 17]byte +var z8524 [1 << 17]byte +var z8525 [1 << 17]byte +var z8526 [1 << 17]byte +var z8527 [1 << 17]byte +var z8528 [1 << 17]byte +var z8529 [1 << 17]byte +var z8530 [1 << 17]byte +var z8531 [1 << 17]byte +var z8532 [1 << 17]byte +var z8533 [1 << 17]byte +var z8534 [1 << 17]byte +var z8535 [1 << 17]byte +var z8536 [1 << 17]byte +var z8537 [1 << 17]byte +var z8538 [1 << 17]byte +var z8539 [1 << 17]byte +var z8540 [1 << 17]byte +var z8541 [1 << 17]byte +var z8542 [1 << 17]byte +var z8543 [1 << 17]byte +var z8544 [1 << 17]byte +var z8545 [1 << 17]byte +var z8546 [1 << 17]byte +var z8547 [1 << 17]byte +var z8548 [1 << 17]byte +var z8549 [1 << 17]byte +var z8550 [1 << 17]byte +var z8551 [1 << 17]byte +var z8552 [1 << 17]byte +var z8553 [1 << 17]byte +var z8554 [1 << 17]byte +var z8555 [1 << 17]byte +var z8556 [1 << 17]byte +var z8557 [1 << 17]byte +var z8558 [1 << 17]byte +var z8559 [1 << 17]byte +var z8560 [1 << 17]byte +var z8561 [1 << 17]byte +var z8562 [1 << 17]byte +var z8563 [1 << 17]byte +var z8564 [1 << 17]byte +var z8565 [1 << 17]byte +var z8566 [1 << 17]byte +var z8567 [1 << 17]byte +var z8568 [1 << 17]byte +var z8569 [1 << 17]byte +var z8570 [1 << 17]byte +var z8571 [1 << 17]byte +var z8572 [1 << 17]byte +var z8573 [1 << 17]byte +var z8574 [1 << 17]byte +var z8575 [1 << 17]byte +var z8576 [1 << 17]byte +var z8577 [1 << 17]byte +var z8578 [1 << 17]byte +var z8579 [1 << 17]byte +var z8580 [1 << 17]byte +var z8581 [1 << 17]byte +var z8582 [1 << 17]byte +var z8583 [1 << 17]byte +var z8584 [1 << 17]byte +var z8585 [1 << 17]byte +var z8586 [1 << 17]byte +var z8587 [1 << 17]byte +var z8588 [1 << 17]byte +var z8589 [1 << 17]byte +var z8590 [1 << 17]byte +var z8591 [1 << 17]byte +var z8592 [1 << 17]byte +var z8593 [1 << 17]byte +var z8594 [1 << 17]byte +var z8595 [1 << 17]byte +var z8596 [1 << 17]byte +var z8597 [1 << 17]byte +var z8598 [1 << 17]byte +var z8599 [1 << 17]byte +var z8600 [1 << 17]byte +var z8601 [1 << 17]byte +var z8602 [1 << 17]byte +var z8603 [1 << 17]byte +var z8604 [1 << 17]byte +var z8605 [1 << 17]byte +var z8606 [1 << 17]byte +var z8607 [1 << 17]byte +var z8608 [1 << 17]byte +var z8609 [1 << 17]byte +var z8610 [1 << 17]byte +var z8611 [1 << 17]byte +var z8612 [1 << 17]byte +var z8613 [1 << 17]byte +var z8614 [1 << 17]byte +var z8615 [1 << 17]byte +var z8616 [1 << 17]byte +var z8617 [1 << 17]byte +var z8618 [1 << 17]byte +var z8619 [1 << 17]byte +var z8620 [1 << 17]byte +var z8621 [1 << 17]byte +var z8622 [1 << 17]byte +var z8623 [1 << 17]byte +var z8624 [1 << 17]byte +var z8625 [1 << 17]byte +var z8626 [1 << 17]byte +var z8627 [1 << 17]byte +var z8628 [1 << 17]byte +var z8629 [1 << 17]byte +var z8630 [1 << 17]byte +var z8631 [1 << 17]byte +var z8632 [1 << 17]byte +var z8633 [1 << 17]byte +var z8634 [1 << 17]byte +var z8635 [1 << 17]byte +var z8636 [1 << 17]byte +var z8637 [1 << 17]byte +var z8638 [1 << 17]byte +var z8639 [1 << 17]byte +var z8640 [1 << 17]byte +var z8641 [1 << 17]byte +var z8642 [1 << 17]byte +var z8643 [1 << 17]byte +var z8644 [1 << 17]byte +var z8645 [1 << 17]byte +var z8646 [1 << 17]byte +var z8647 [1 << 17]byte +var z8648 [1 << 17]byte +var z8649 [1 << 17]byte +var z8650 [1 << 17]byte +var z8651 [1 << 17]byte +var z8652 [1 << 17]byte +var z8653 [1 << 17]byte +var z8654 [1 << 17]byte +var z8655 [1 << 17]byte +var z8656 [1 << 17]byte +var z8657 [1 << 17]byte +var z8658 [1 << 17]byte +var z8659 [1 << 17]byte +var z8660 [1 << 17]byte +var z8661 [1 << 17]byte +var z8662 [1 << 17]byte +var z8663 [1 << 17]byte +var z8664 [1 << 17]byte +var z8665 [1 << 17]byte +var z8666 [1 << 17]byte +var z8667 [1 << 17]byte +var z8668 [1 << 17]byte +var z8669 [1 << 17]byte +var z8670 [1 << 17]byte +var z8671 [1 << 17]byte +var z8672 [1 << 17]byte +var z8673 [1 << 17]byte +var z8674 [1 << 17]byte +var z8675 [1 << 17]byte +var z8676 [1 << 17]byte +var z8677 [1 << 17]byte +var z8678 [1 << 17]byte +var z8679 [1 << 17]byte +var z8680 [1 << 17]byte +var z8681 [1 << 17]byte +var z8682 [1 << 17]byte +var z8683 [1 << 17]byte +var z8684 [1 << 17]byte +var z8685 [1 << 17]byte +var z8686 [1 << 17]byte +var z8687 [1 << 17]byte +var z8688 [1 << 17]byte +var z8689 [1 << 17]byte +var z8690 [1 << 17]byte +var z8691 [1 << 17]byte +var z8692 [1 << 17]byte +var z8693 [1 << 17]byte +var z8694 [1 << 17]byte +var z8695 [1 << 17]byte +var z8696 [1 << 17]byte +var z8697 [1 << 17]byte +var z8698 [1 << 17]byte +var z8699 [1 << 17]byte +var z8700 [1 << 17]byte +var z8701 [1 << 17]byte +var z8702 [1 << 17]byte +var z8703 [1 << 17]byte +var z8704 [1 << 17]byte +var z8705 [1 << 17]byte +var z8706 [1 << 17]byte +var z8707 [1 << 17]byte +var z8708 [1 << 17]byte +var z8709 [1 << 17]byte +var z8710 [1 << 17]byte +var z8711 [1 << 17]byte +var z8712 [1 << 17]byte +var z8713 [1 << 17]byte +var z8714 [1 << 17]byte +var z8715 [1 << 17]byte +var z8716 [1 << 17]byte +var z8717 [1 << 17]byte +var z8718 [1 << 17]byte +var z8719 [1 << 17]byte +var z8720 [1 << 17]byte +var z8721 [1 << 17]byte +var z8722 [1 << 17]byte +var z8723 [1 << 17]byte +var z8724 [1 << 17]byte +var z8725 [1 << 17]byte +var z8726 [1 << 17]byte +var z8727 [1 << 17]byte +var z8728 [1 << 17]byte +var z8729 [1 << 17]byte +var z8730 [1 << 17]byte +var z8731 [1 << 17]byte +var z8732 [1 << 17]byte +var z8733 [1 << 17]byte +var z8734 [1 << 17]byte +var z8735 [1 << 17]byte +var z8736 [1 << 17]byte +var z8737 [1 << 17]byte +var z8738 [1 << 17]byte +var z8739 [1 << 17]byte +var z8740 [1 << 17]byte +var z8741 [1 << 17]byte +var z8742 [1 << 17]byte +var z8743 [1 << 17]byte +var z8744 [1 << 17]byte +var z8745 [1 << 17]byte +var z8746 [1 << 17]byte +var z8747 [1 << 17]byte +var z8748 [1 << 17]byte +var z8749 [1 << 17]byte +var z8750 [1 << 17]byte +var z8751 [1 << 17]byte +var z8752 [1 << 17]byte +var z8753 [1 << 17]byte +var z8754 [1 << 17]byte +var z8755 [1 << 17]byte +var z8756 [1 << 17]byte +var z8757 [1 << 17]byte +var z8758 [1 << 17]byte +var z8759 [1 << 17]byte +var z8760 [1 << 17]byte +var z8761 [1 << 17]byte +var z8762 [1 << 17]byte +var z8763 [1 << 17]byte +var z8764 [1 << 17]byte +var z8765 [1 << 17]byte +var z8766 [1 << 17]byte +var z8767 [1 << 17]byte +var z8768 [1 << 17]byte +var z8769 [1 << 17]byte +var z8770 [1 << 17]byte +var z8771 [1 << 17]byte +var z8772 [1 << 17]byte +var z8773 [1 << 17]byte +var z8774 [1 << 17]byte +var z8775 [1 << 17]byte +var z8776 [1 << 17]byte +var z8777 [1 << 17]byte +var z8778 [1 << 17]byte +var z8779 [1 << 17]byte +var z8780 [1 << 17]byte +var z8781 [1 << 17]byte +var z8782 [1 << 17]byte +var z8783 [1 << 17]byte +var z8784 [1 << 17]byte +var z8785 [1 << 17]byte +var z8786 [1 << 17]byte +var z8787 [1 << 17]byte +var z8788 [1 << 17]byte +var z8789 [1 << 17]byte +var z8790 [1 << 17]byte +var z8791 [1 << 17]byte +var z8792 [1 << 17]byte +var z8793 [1 << 17]byte +var z8794 [1 << 17]byte +var z8795 [1 << 17]byte +var z8796 [1 << 17]byte +var z8797 [1 << 17]byte +var z8798 [1 << 17]byte +var z8799 [1 << 17]byte +var z8800 [1 << 17]byte +var z8801 [1 << 17]byte +var z8802 [1 << 17]byte +var z8803 [1 << 17]byte +var z8804 [1 << 17]byte +var z8805 [1 << 17]byte +var z8806 [1 << 17]byte +var z8807 [1 << 17]byte +var z8808 [1 << 17]byte +var z8809 [1 << 17]byte +var z8810 [1 << 17]byte +var z8811 [1 << 17]byte +var z8812 [1 << 17]byte +var z8813 [1 << 17]byte +var z8814 [1 << 17]byte +var z8815 [1 << 17]byte +var z8816 [1 << 17]byte +var z8817 [1 << 17]byte +var z8818 [1 << 17]byte +var z8819 [1 << 17]byte +var z8820 [1 << 17]byte +var z8821 [1 << 17]byte +var z8822 [1 << 17]byte +var z8823 [1 << 17]byte +var z8824 [1 << 17]byte +var z8825 [1 << 17]byte +var z8826 [1 << 17]byte +var z8827 [1 << 17]byte +var z8828 [1 << 17]byte +var z8829 [1 << 17]byte +var z8830 [1 << 17]byte +var z8831 [1 << 17]byte +var z8832 [1 << 17]byte +var z8833 [1 << 17]byte +var z8834 [1 << 17]byte +var z8835 [1 << 17]byte +var z8836 [1 << 17]byte +var z8837 [1 << 17]byte +var z8838 [1 << 17]byte +var z8839 [1 << 17]byte +var z8840 [1 << 17]byte +var z8841 [1 << 17]byte +var z8842 [1 << 17]byte +var z8843 [1 << 17]byte +var z8844 [1 << 17]byte +var z8845 [1 << 17]byte +var z8846 [1 << 17]byte +var z8847 [1 << 17]byte +var z8848 [1 << 17]byte +var z8849 [1 << 17]byte +var z8850 [1 << 17]byte +var z8851 [1 << 17]byte +var z8852 [1 << 17]byte +var z8853 [1 << 17]byte +var z8854 [1 << 17]byte +var z8855 [1 << 17]byte +var z8856 [1 << 17]byte +var z8857 [1 << 17]byte +var z8858 [1 << 17]byte +var z8859 [1 << 17]byte +var z8860 [1 << 17]byte +var z8861 [1 << 17]byte +var z8862 [1 << 17]byte +var z8863 [1 << 17]byte +var z8864 [1 << 17]byte +var z8865 [1 << 17]byte +var z8866 [1 << 17]byte +var z8867 [1 << 17]byte +var z8868 [1 << 17]byte +var z8869 [1 << 17]byte +var z8870 [1 << 17]byte +var z8871 [1 << 17]byte +var z8872 [1 << 17]byte +var z8873 [1 << 17]byte +var z8874 [1 << 17]byte +var z8875 [1 << 17]byte +var z8876 [1 << 17]byte +var z8877 [1 << 17]byte +var z8878 [1 << 17]byte +var z8879 [1 << 17]byte +var z8880 [1 << 17]byte +var z8881 [1 << 17]byte +var z8882 [1 << 17]byte +var z8883 [1 << 17]byte +var z8884 [1 << 17]byte +var z8885 [1 << 17]byte +var z8886 [1 << 17]byte +var z8887 [1 << 17]byte +var z8888 [1 << 17]byte +var z8889 [1 << 17]byte +var z8890 [1 << 17]byte +var z8891 [1 << 17]byte +var z8892 [1 << 17]byte +var z8893 [1 << 17]byte +var z8894 [1 << 17]byte +var z8895 [1 << 17]byte +var z8896 [1 << 17]byte +var z8897 [1 << 17]byte +var z8898 [1 << 17]byte +var z8899 [1 << 17]byte +var z8900 [1 << 17]byte +var z8901 [1 << 17]byte +var z8902 [1 << 17]byte +var z8903 [1 << 17]byte +var z8904 [1 << 17]byte +var z8905 [1 << 17]byte +var z8906 [1 << 17]byte +var z8907 [1 << 17]byte +var z8908 [1 << 17]byte +var z8909 [1 << 17]byte +var z8910 [1 << 17]byte +var z8911 [1 << 17]byte +var z8912 [1 << 17]byte +var z8913 [1 << 17]byte +var z8914 [1 << 17]byte +var z8915 [1 << 17]byte +var z8916 [1 << 17]byte +var z8917 [1 << 17]byte +var z8918 [1 << 17]byte +var z8919 [1 << 17]byte +var z8920 [1 << 17]byte +var z8921 [1 << 17]byte +var z8922 [1 << 17]byte +var z8923 [1 << 17]byte +var z8924 [1 << 17]byte +var z8925 [1 << 17]byte +var z8926 [1 << 17]byte +var z8927 [1 << 17]byte +var z8928 [1 << 17]byte +var z8929 [1 << 17]byte +var z8930 [1 << 17]byte +var z8931 [1 << 17]byte +var z8932 [1 << 17]byte +var z8933 [1 << 17]byte +var z8934 [1 << 17]byte +var z8935 [1 << 17]byte +var z8936 [1 << 17]byte +var z8937 [1 << 17]byte +var z8938 [1 << 17]byte +var z8939 [1 << 17]byte +var z8940 [1 << 17]byte +var z8941 [1 << 17]byte +var z8942 [1 << 17]byte +var z8943 [1 << 17]byte +var z8944 [1 << 17]byte +var z8945 [1 << 17]byte +var z8946 [1 << 17]byte +var z8947 [1 << 17]byte +var z8948 [1 << 17]byte +var z8949 [1 << 17]byte +var z8950 [1 << 17]byte +var z8951 [1 << 17]byte +var z8952 [1 << 17]byte +var z8953 [1 << 17]byte +var z8954 [1 << 17]byte +var z8955 [1 << 17]byte +var z8956 [1 << 17]byte +var z8957 [1 << 17]byte +var z8958 [1 << 17]byte +var z8959 [1 << 17]byte +var z8960 [1 << 17]byte +var z8961 [1 << 17]byte +var z8962 [1 << 17]byte +var z8963 [1 << 17]byte +var z8964 [1 << 17]byte +var z8965 [1 << 17]byte +var z8966 [1 << 17]byte +var z8967 [1 << 17]byte +var z8968 [1 << 17]byte +var z8969 [1 << 17]byte +var z8970 [1 << 17]byte +var z8971 [1 << 17]byte +var z8972 [1 << 17]byte +var z8973 [1 << 17]byte +var z8974 [1 << 17]byte +var z8975 [1 << 17]byte +var z8976 [1 << 17]byte +var z8977 [1 << 17]byte +var z8978 [1 << 17]byte +var z8979 [1 << 17]byte +var z8980 [1 << 17]byte +var z8981 [1 << 17]byte +var z8982 [1 << 17]byte +var z8983 [1 << 17]byte +var z8984 [1 << 17]byte +var z8985 [1 << 17]byte +var z8986 [1 << 17]byte +var z8987 [1 << 17]byte +var z8988 [1 << 17]byte +var z8989 [1 << 17]byte +var z8990 [1 << 17]byte +var z8991 [1 << 17]byte +var z8992 [1 << 17]byte +var z8993 [1 << 17]byte +var z8994 [1 << 17]byte +var z8995 [1 << 17]byte +var z8996 [1 << 17]byte +var z8997 [1 << 17]byte +var z8998 [1 << 17]byte +var z8999 [1 << 17]byte +var z9000 [1 << 17]byte +var z9001 [1 << 17]byte +var z9002 [1 << 17]byte +var z9003 [1 << 17]byte +var z9004 [1 << 17]byte +var z9005 [1 << 17]byte +var z9006 [1 << 17]byte +var z9007 [1 << 17]byte +var z9008 [1 << 17]byte +var z9009 [1 << 17]byte +var z9010 [1 << 17]byte +var z9011 [1 << 17]byte +var z9012 [1 << 17]byte +var z9013 [1 << 17]byte +var z9014 [1 << 17]byte +var z9015 [1 << 17]byte +var z9016 [1 << 17]byte +var z9017 [1 << 17]byte +var z9018 [1 << 17]byte +var z9019 [1 << 17]byte +var z9020 [1 << 17]byte +var z9021 [1 << 17]byte +var z9022 [1 << 17]byte +var z9023 [1 << 17]byte +var z9024 [1 << 17]byte +var z9025 [1 << 17]byte +var z9026 [1 << 17]byte +var z9027 [1 << 17]byte +var z9028 [1 << 17]byte +var z9029 [1 << 17]byte +var z9030 [1 << 17]byte +var z9031 [1 << 17]byte +var z9032 [1 << 17]byte +var z9033 [1 << 17]byte +var z9034 [1 << 17]byte +var z9035 [1 << 17]byte +var z9036 [1 << 17]byte +var z9037 [1 << 17]byte +var z9038 [1 << 17]byte +var z9039 [1 << 17]byte +var z9040 [1 << 17]byte +var z9041 [1 << 17]byte +var z9042 [1 << 17]byte +var z9043 [1 << 17]byte +var z9044 [1 << 17]byte +var z9045 [1 << 17]byte +var z9046 [1 << 17]byte +var z9047 [1 << 17]byte +var z9048 [1 << 17]byte +var z9049 [1 << 17]byte +var z9050 [1 << 17]byte +var z9051 [1 << 17]byte +var z9052 [1 << 17]byte +var z9053 [1 << 17]byte +var z9054 [1 << 17]byte +var z9055 [1 << 17]byte +var z9056 [1 << 17]byte +var z9057 [1 << 17]byte +var z9058 [1 << 17]byte +var z9059 [1 << 17]byte +var z9060 [1 << 17]byte +var z9061 [1 << 17]byte +var z9062 [1 << 17]byte +var z9063 [1 << 17]byte +var z9064 [1 << 17]byte +var z9065 [1 << 17]byte +var z9066 [1 << 17]byte +var z9067 [1 << 17]byte +var z9068 [1 << 17]byte +var z9069 [1 << 17]byte +var z9070 [1 << 17]byte +var z9071 [1 << 17]byte +var z9072 [1 << 17]byte +var z9073 [1 << 17]byte +var z9074 [1 << 17]byte +var z9075 [1 << 17]byte +var z9076 [1 << 17]byte +var z9077 [1 << 17]byte +var z9078 [1 << 17]byte +var z9079 [1 << 17]byte +var z9080 [1 << 17]byte +var z9081 [1 << 17]byte +var z9082 [1 << 17]byte +var z9083 [1 << 17]byte +var z9084 [1 << 17]byte +var z9085 [1 << 17]byte +var z9086 [1 << 17]byte +var z9087 [1 << 17]byte +var z9088 [1 << 17]byte +var z9089 [1 << 17]byte +var z9090 [1 << 17]byte +var z9091 [1 << 17]byte +var z9092 [1 << 17]byte +var z9093 [1 << 17]byte +var z9094 [1 << 17]byte +var z9095 [1 << 17]byte +var z9096 [1 << 17]byte +var z9097 [1 << 17]byte +var z9098 [1 << 17]byte +var z9099 [1 << 17]byte +var z9100 [1 << 17]byte +var z9101 [1 << 17]byte +var z9102 [1 << 17]byte +var z9103 [1 << 17]byte +var z9104 [1 << 17]byte +var z9105 [1 << 17]byte +var z9106 [1 << 17]byte +var z9107 [1 << 17]byte +var z9108 [1 << 17]byte +var z9109 [1 << 17]byte +var z9110 [1 << 17]byte +var z9111 [1 << 17]byte +var z9112 [1 << 17]byte +var z9113 [1 << 17]byte +var z9114 [1 << 17]byte +var z9115 [1 << 17]byte +var z9116 [1 << 17]byte +var z9117 [1 << 17]byte +var z9118 [1 << 17]byte +var z9119 [1 << 17]byte +var z9120 [1 << 17]byte +var z9121 [1 << 17]byte +var z9122 [1 << 17]byte +var z9123 [1 << 17]byte +var z9124 [1 << 17]byte +var z9125 [1 << 17]byte +var z9126 [1 << 17]byte +var z9127 [1 << 17]byte +var z9128 [1 << 17]byte +var z9129 [1 << 17]byte +var z9130 [1 << 17]byte +var z9131 [1 << 17]byte +var z9132 [1 << 17]byte +var z9133 [1 << 17]byte +var z9134 [1 << 17]byte +var z9135 [1 << 17]byte +var z9136 [1 << 17]byte +var z9137 [1 << 17]byte +var z9138 [1 << 17]byte +var z9139 [1 << 17]byte +var z9140 [1 << 17]byte +var z9141 [1 << 17]byte +var z9142 [1 << 17]byte +var z9143 [1 << 17]byte +var z9144 [1 << 17]byte +var z9145 [1 << 17]byte +var z9146 [1 << 17]byte +var z9147 [1 << 17]byte +var z9148 [1 << 17]byte +var z9149 [1 << 17]byte +var z9150 [1 << 17]byte +var z9151 [1 << 17]byte +var z9152 [1 << 17]byte +var z9153 [1 << 17]byte +var z9154 [1 << 17]byte +var z9155 [1 << 17]byte +var z9156 [1 << 17]byte +var z9157 [1 << 17]byte +var z9158 [1 << 17]byte +var z9159 [1 << 17]byte +var z9160 [1 << 17]byte +var z9161 [1 << 17]byte +var z9162 [1 << 17]byte +var z9163 [1 << 17]byte +var z9164 [1 << 17]byte +var z9165 [1 << 17]byte +var z9166 [1 << 17]byte +var z9167 [1 << 17]byte +var z9168 [1 << 17]byte +var z9169 [1 << 17]byte +var z9170 [1 << 17]byte +var z9171 [1 << 17]byte +var z9172 [1 << 17]byte +var z9173 [1 << 17]byte +var z9174 [1 << 17]byte +var z9175 [1 << 17]byte +var z9176 [1 << 17]byte +var z9177 [1 << 17]byte +var z9178 [1 << 17]byte +var z9179 [1 << 17]byte +var z9180 [1 << 17]byte +var z9181 [1 << 17]byte +var z9182 [1 << 17]byte +var z9183 [1 << 17]byte +var z9184 [1 << 17]byte +var z9185 [1 << 17]byte +var z9186 [1 << 17]byte +var z9187 [1 << 17]byte +var z9188 [1 << 17]byte +var z9189 [1 << 17]byte +var z9190 [1 << 17]byte +var z9191 [1 << 17]byte +var z9192 [1 << 17]byte +var z9193 [1 << 17]byte +var z9194 [1 << 17]byte +var z9195 [1 << 17]byte +var z9196 [1 << 17]byte +var z9197 [1 << 17]byte +var z9198 [1 << 17]byte +var z9199 [1 << 17]byte +var z9200 [1 << 17]byte +var z9201 [1 << 17]byte +var z9202 [1 << 17]byte +var z9203 [1 << 17]byte +var z9204 [1 << 17]byte +var z9205 [1 << 17]byte +var z9206 [1 << 17]byte +var z9207 [1 << 17]byte +var z9208 [1 << 17]byte +var z9209 [1 << 17]byte +var z9210 [1 << 17]byte +var z9211 [1 << 17]byte +var z9212 [1 << 17]byte +var z9213 [1 << 17]byte +var z9214 [1 << 17]byte +var z9215 [1 << 17]byte +var z9216 [1 << 17]byte +var z9217 [1 << 17]byte +var z9218 [1 << 17]byte +var z9219 [1 << 17]byte +var z9220 [1 << 17]byte +var z9221 [1 << 17]byte +var z9222 [1 << 17]byte +var z9223 [1 << 17]byte +var z9224 [1 << 17]byte +var z9225 [1 << 17]byte +var z9226 [1 << 17]byte +var z9227 [1 << 17]byte +var z9228 [1 << 17]byte +var z9229 [1 << 17]byte +var z9230 [1 << 17]byte +var z9231 [1 << 17]byte +var z9232 [1 << 17]byte +var z9233 [1 << 17]byte +var z9234 [1 << 17]byte +var z9235 [1 << 17]byte +var z9236 [1 << 17]byte +var z9237 [1 << 17]byte +var z9238 [1 << 17]byte +var z9239 [1 << 17]byte +var z9240 [1 << 17]byte +var z9241 [1 << 17]byte +var z9242 [1 << 17]byte +var z9243 [1 << 17]byte +var z9244 [1 << 17]byte +var z9245 [1 << 17]byte +var z9246 [1 << 17]byte +var z9247 [1 << 17]byte +var z9248 [1 << 17]byte +var z9249 [1 << 17]byte +var z9250 [1 << 17]byte +var z9251 [1 << 17]byte +var z9252 [1 << 17]byte +var z9253 [1 << 17]byte +var z9254 [1 << 17]byte +var z9255 [1 << 17]byte +var z9256 [1 << 17]byte +var z9257 [1 << 17]byte +var z9258 [1 << 17]byte +var z9259 [1 << 17]byte +var z9260 [1 << 17]byte +var z9261 [1 << 17]byte +var z9262 [1 << 17]byte +var z9263 [1 << 17]byte +var z9264 [1 << 17]byte +var z9265 [1 << 17]byte +var z9266 [1 << 17]byte +var z9267 [1 << 17]byte +var z9268 [1 << 17]byte +var z9269 [1 << 17]byte +var z9270 [1 << 17]byte +var z9271 [1 << 17]byte +var z9272 [1 << 17]byte +var z9273 [1 << 17]byte +var z9274 [1 << 17]byte +var z9275 [1 << 17]byte +var z9276 [1 << 17]byte +var z9277 [1 << 17]byte +var z9278 [1 << 17]byte +var z9279 [1 << 17]byte +var z9280 [1 << 17]byte +var z9281 [1 << 17]byte +var z9282 [1 << 17]byte +var z9283 [1 << 17]byte +var z9284 [1 << 17]byte +var z9285 [1 << 17]byte +var z9286 [1 << 17]byte +var z9287 [1 << 17]byte +var z9288 [1 << 17]byte +var z9289 [1 << 17]byte +var z9290 [1 << 17]byte +var z9291 [1 << 17]byte +var z9292 [1 << 17]byte +var z9293 [1 << 17]byte +var z9294 [1 << 17]byte +var z9295 [1 << 17]byte +var z9296 [1 << 17]byte +var z9297 [1 << 17]byte +var z9298 [1 << 17]byte +var z9299 [1 << 17]byte +var z9300 [1 << 17]byte +var z9301 [1 << 17]byte +var z9302 [1 << 17]byte +var z9303 [1 << 17]byte +var z9304 [1 << 17]byte +var z9305 [1 << 17]byte +var z9306 [1 << 17]byte +var z9307 [1 << 17]byte +var z9308 [1 << 17]byte +var z9309 [1 << 17]byte +var z9310 [1 << 17]byte +var z9311 [1 << 17]byte +var z9312 [1 << 17]byte +var z9313 [1 << 17]byte +var z9314 [1 << 17]byte +var z9315 [1 << 17]byte +var z9316 [1 << 17]byte +var z9317 [1 << 17]byte +var z9318 [1 << 17]byte +var z9319 [1 << 17]byte +var z9320 [1 << 17]byte +var z9321 [1 << 17]byte +var z9322 [1 << 17]byte +var z9323 [1 << 17]byte +var z9324 [1 << 17]byte +var z9325 [1 << 17]byte +var z9326 [1 << 17]byte +var z9327 [1 << 17]byte +var z9328 [1 << 17]byte +var z9329 [1 << 17]byte +var z9330 [1 << 17]byte +var z9331 [1 << 17]byte +var z9332 [1 << 17]byte +var z9333 [1 << 17]byte +var z9334 [1 << 17]byte +var z9335 [1 << 17]byte +var z9336 [1 << 17]byte +var z9337 [1 << 17]byte +var z9338 [1 << 17]byte +var z9339 [1 << 17]byte +var z9340 [1 << 17]byte +var z9341 [1 << 17]byte +var z9342 [1 << 17]byte +var z9343 [1 << 17]byte +var z9344 [1 << 17]byte +var z9345 [1 << 17]byte +var z9346 [1 << 17]byte +var z9347 [1 << 17]byte +var z9348 [1 << 17]byte +var z9349 [1 << 17]byte +var z9350 [1 << 17]byte +var z9351 [1 << 17]byte +var z9352 [1 << 17]byte +var z9353 [1 << 17]byte +var z9354 [1 << 17]byte +var z9355 [1 << 17]byte +var z9356 [1 << 17]byte +var z9357 [1 << 17]byte +var z9358 [1 << 17]byte +var z9359 [1 << 17]byte +var z9360 [1 << 17]byte +var z9361 [1 << 17]byte +var z9362 [1 << 17]byte +var z9363 [1 << 17]byte +var z9364 [1 << 17]byte +var z9365 [1 << 17]byte +var z9366 [1 << 17]byte +var z9367 [1 << 17]byte +var z9368 [1 << 17]byte +var z9369 [1 << 17]byte +var z9370 [1 << 17]byte +var z9371 [1 << 17]byte +var z9372 [1 << 17]byte +var z9373 [1 << 17]byte +var z9374 [1 << 17]byte +var z9375 [1 << 17]byte +var z9376 [1 << 17]byte +var z9377 [1 << 17]byte +var z9378 [1 << 17]byte +var z9379 [1 << 17]byte +var z9380 [1 << 17]byte +var z9381 [1 << 17]byte +var z9382 [1 << 17]byte +var z9383 [1 << 17]byte +var z9384 [1 << 17]byte +var z9385 [1 << 17]byte +var z9386 [1 << 17]byte +var z9387 [1 << 17]byte +var z9388 [1 << 17]byte +var z9389 [1 << 17]byte +var z9390 [1 << 17]byte +var z9391 [1 << 17]byte +var z9392 [1 << 17]byte +var z9393 [1 << 17]byte +var z9394 [1 << 17]byte +var z9395 [1 << 17]byte +var z9396 [1 << 17]byte +var z9397 [1 << 17]byte +var z9398 [1 << 17]byte +var z9399 [1 << 17]byte +var z9400 [1 << 17]byte +var z9401 [1 << 17]byte +var z9402 [1 << 17]byte +var z9403 [1 << 17]byte +var z9404 [1 << 17]byte +var z9405 [1 << 17]byte +var z9406 [1 << 17]byte +var z9407 [1 << 17]byte +var z9408 [1 << 17]byte +var z9409 [1 << 17]byte +var z9410 [1 << 17]byte +var z9411 [1 << 17]byte +var z9412 [1 << 17]byte +var z9413 [1 << 17]byte +var z9414 [1 << 17]byte +var z9415 [1 << 17]byte +var z9416 [1 << 17]byte +var z9417 [1 << 17]byte +var z9418 [1 << 17]byte +var z9419 [1 << 17]byte +var z9420 [1 << 17]byte +var z9421 [1 << 17]byte +var z9422 [1 << 17]byte +var z9423 [1 << 17]byte +var z9424 [1 << 17]byte +var z9425 [1 << 17]byte +var z9426 [1 << 17]byte +var z9427 [1 << 17]byte +var z9428 [1 << 17]byte +var z9429 [1 << 17]byte +var z9430 [1 << 17]byte +var z9431 [1 << 17]byte +var z9432 [1 << 17]byte +var z9433 [1 << 17]byte +var z9434 [1 << 17]byte +var z9435 [1 << 17]byte +var z9436 [1 << 17]byte +var z9437 [1 << 17]byte +var z9438 [1 << 17]byte +var z9439 [1 << 17]byte +var z9440 [1 << 17]byte +var z9441 [1 << 17]byte +var z9442 [1 << 17]byte +var z9443 [1 << 17]byte +var z9444 [1 << 17]byte +var z9445 [1 << 17]byte +var z9446 [1 << 17]byte +var z9447 [1 << 17]byte +var z9448 [1 << 17]byte +var z9449 [1 << 17]byte +var z9450 [1 << 17]byte +var z9451 [1 << 17]byte +var z9452 [1 << 17]byte +var z9453 [1 << 17]byte +var z9454 [1 << 17]byte +var z9455 [1 << 17]byte +var z9456 [1 << 17]byte +var z9457 [1 << 17]byte +var z9458 [1 << 17]byte +var z9459 [1 << 17]byte +var z9460 [1 << 17]byte +var z9461 [1 << 17]byte +var z9462 [1 << 17]byte +var z9463 [1 << 17]byte +var z9464 [1 << 17]byte +var z9465 [1 << 17]byte +var z9466 [1 << 17]byte +var z9467 [1 << 17]byte +var z9468 [1 << 17]byte +var z9469 [1 << 17]byte +var z9470 [1 << 17]byte +var z9471 [1 << 17]byte +var z9472 [1 << 17]byte +var z9473 [1 << 17]byte +var z9474 [1 << 17]byte +var z9475 [1 << 17]byte +var z9476 [1 << 17]byte +var z9477 [1 << 17]byte +var z9478 [1 << 17]byte +var z9479 [1 << 17]byte +var z9480 [1 << 17]byte +var z9481 [1 << 17]byte +var z9482 [1 << 17]byte +var z9483 [1 << 17]byte +var z9484 [1 << 17]byte +var z9485 [1 << 17]byte +var z9486 [1 << 17]byte +var z9487 [1 << 17]byte +var z9488 [1 << 17]byte +var z9489 [1 << 17]byte +var z9490 [1 << 17]byte +var z9491 [1 << 17]byte +var z9492 [1 << 17]byte +var z9493 [1 << 17]byte +var z9494 [1 << 17]byte +var z9495 [1 << 17]byte +var z9496 [1 << 17]byte +var z9497 [1 << 17]byte +var z9498 [1 << 17]byte +var z9499 [1 << 17]byte +var z9500 [1 << 17]byte +var z9501 [1 << 17]byte +var z9502 [1 << 17]byte +var z9503 [1 << 17]byte +var z9504 [1 << 17]byte +var z9505 [1 << 17]byte +var z9506 [1 << 17]byte +var z9507 [1 << 17]byte +var z9508 [1 << 17]byte +var z9509 [1 << 17]byte +var z9510 [1 << 17]byte +var z9511 [1 << 17]byte +var z9512 [1 << 17]byte +var z9513 [1 << 17]byte +var z9514 [1 << 17]byte +var z9515 [1 << 17]byte +var z9516 [1 << 17]byte +var z9517 [1 << 17]byte +var z9518 [1 << 17]byte +var z9519 [1 << 17]byte +var z9520 [1 << 17]byte +var z9521 [1 << 17]byte +var z9522 [1 << 17]byte +var z9523 [1 << 17]byte +var z9524 [1 << 17]byte +var z9525 [1 << 17]byte +var z9526 [1 << 17]byte +var z9527 [1 << 17]byte +var z9528 [1 << 17]byte +var z9529 [1 << 17]byte +var z9530 [1 << 17]byte +var z9531 [1 << 17]byte +var z9532 [1 << 17]byte +var z9533 [1 << 17]byte +var z9534 [1 << 17]byte +var z9535 [1 << 17]byte +var z9536 [1 << 17]byte +var z9537 [1 << 17]byte +var z9538 [1 << 17]byte +var z9539 [1 << 17]byte +var z9540 [1 << 17]byte +var z9541 [1 << 17]byte +var z9542 [1 << 17]byte +var z9543 [1 << 17]byte +var z9544 [1 << 17]byte +var z9545 [1 << 17]byte +var z9546 [1 << 17]byte +var z9547 [1 << 17]byte +var z9548 [1 << 17]byte +var z9549 [1 << 17]byte +var z9550 [1 << 17]byte +var z9551 [1 << 17]byte +var z9552 [1 << 17]byte +var z9553 [1 << 17]byte +var z9554 [1 << 17]byte +var z9555 [1 << 17]byte +var z9556 [1 << 17]byte +var z9557 [1 << 17]byte +var z9558 [1 << 17]byte +var z9559 [1 << 17]byte +var z9560 [1 << 17]byte +var z9561 [1 << 17]byte +var z9562 [1 << 17]byte +var z9563 [1 << 17]byte +var z9564 [1 << 17]byte +var z9565 [1 << 17]byte +var z9566 [1 << 17]byte +var z9567 [1 << 17]byte +var z9568 [1 << 17]byte +var z9569 [1 << 17]byte +var z9570 [1 << 17]byte +var z9571 [1 << 17]byte +var z9572 [1 << 17]byte +var z9573 [1 << 17]byte +var z9574 [1 << 17]byte +var z9575 [1 << 17]byte +var z9576 [1 << 17]byte +var z9577 [1 << 17]byte +var z9578 [1 << 17]byte +var z9579 [1 << 17]byte +var z9580 [1 << 17]byte +var z9581 [1 << 17]byte +var z9582 [1 << 17]byte +var z9583 [1 << 17]byte +var z9584 [1 << 17]byte +var z9585 [1 << 17]byte +var z9586 [1 << 17]byte +var z9587 [1 << 17]byte +var z9588 [1 << 17]byte +var z9589 [1 << 17]byte +var z9590 [1 << 17]byte +var z9591 [1 << 17]byte +var z9592 [1 << 17]byte +var z9593 [1 << 17]byte +var z9594 [1 << 17]byte +var z9595 [1 << 17]byte +var z9596 [1 << 17]byte +var z9597 [1 << 17]byte +var z9598 [1 << 17]byte +var z9599 [1 << 17]byte +var z9600 [1 << 17]byte +var z9601 [1 << 17]byte +var z9602 [1 << 17]byte +var z9603 [1 << 17]byte +var z9604 [1 << 17]byte +var z9605 [1 << 17]byte +var z9606 [1 << 17]byte +var z9607 [1 << 17]byte +var z9608 [1 << 17]byte +var z9609 [1 << 17]byte +var z9610 [1 << 17]byte +var z9611 [1 << 17]byte +var z9612 [1 << 17]byte +var z9613 [1 << 17]byte +var z9614 [1 << 17]byte +var z9615 [1 << 17]byte +var z9616 [1 << 17]byte +var z9617 [1 << 17]byte +var z9618 [1 << 17]byte +var z9619 [1 << 17]byte +var z9620 [1 << 17]byte +var z9621 [1 << 17]byte +var z9622 [1 << 17]byte +var z9623 [1 << 17]byte +var z9624 [1 << 17]byte +var z9625 [1 << 17]byte +var z9626 [1 << 17]byte +var z9627 [1 << 17]byte +var z9628 [1 << 17]byte +var z9629 [1 << 17]byte +var z9630 [1 << 17]byte +var z9631 [1 << 17]byte +var z9632 [1 << 17]byte +var z9633 [1 << 17]byte +var z9634 [1 << 17]byte +var z9635 [1 << 17]byte +var z9636 [1 << 17]byte +var z9637 [1 << 17]byte +var z9638 [1 << 17]byte +var z9639 [1 << 17]byte +var z9640 [1 << 17]byte +var z9641 [1 << 17]byte +var z9642 [1 << 17]byte +var z9643 [1 << 17]byte +var z9644 [1 << 17]byte +var z9645 [1 << 17]byte +var z9646 [1 << 17]byte +var z9647 [1 << 17]byte +var z9648 [1 << 17]byte +var z9649 [1 << 17]byte +var z9650 [1 << 17]byte +var z9651 [1 << 17]byte +var z9652 [1 << 17]byte +var z9653 [1 << 17]byte +var z9654 [1 << 17]byte +var z9655 [1 << 17]byte +var z9656 [1 << 17]byte +var z9657 [1 << 17]byte +var z9658 [1 << 17]byte +var z9659 [1 << 17]byte +var z9660 [1 << 17]byte +var z9661 [1 << 17]byte +var z9662 [1 << 17]byte +var z9663 [1 << 17]byte +var z9664 [1 << 17]byte +var z9665 [1 << 17]byte +var z9666 [1 << 17]byte +var z9667 [1 << 17]byte +var z9668 [1 << 17]byte +var z9669 [1 << 17]byte +var z9670 [1 << 17]byte +var z9671 [1 << 17]byte +var z9672 [1 << 17]byte +var z9673 [1 << 17]byte +var z9674 [1 << 17]byte +var z9675 [1 << 17]byte +var z9676 [1 << 17]byte +var z9677 [1 << 17]byte +var z9678 [1 << 17]byte +var z9679 [1 << 17]byte +var z9680 [1 << 17]byte +var z9681 [1 << 17]byte +var z9682 [1 << 17]byte +var z9683 [1 << 17]byte +var z9684 [1 << 17]byte +var z9685 [1 << 17]byte +var z9686 [1 << 17]byte +var z9687 [1 << 17]byte +var z9688 [1 << 17]byte +var z9689 [1 << 17]byte +var z9690 [1 << 17]byte +var z9691 [1 << 17]byte +var z9692 [1 << 17]byte +var z9693 [1 << 17]byte +var z9694 [1 << 17]byte +var z9695 [1 << 17]byte +var z9696 [1 << 17]byte +var z9697 [1 << 17]byte +var z9698 [1 << 17]byte +var z9699 [1 << 17]byte +var z9700 [1 << 17]byte +var z9701 [1 << 17]byte +var z9702 [1 << 17]byte +var z9703 [1 << 17]byte +var z9704 [1 << 17]byte +var z9705 [1 << 17]byte +var z9706 [1 << 17]byte +var z9707 [1 << 17]byte +var z9708 [1 << 17]byte +var z9709 [1 << 17]byte +var z9710 [1 << 17]byte +var z9711 [1 << 17]byte +var z9712 [1 << 17]byte +var z9713 [1 << 17]byte +var z9714 [1 << 17]byte +var z9715 [1 << 17]byte +var z9716 [1 << 17]byte +var z9717 [1 << 17]byte +var z9718 [1 << 17]byte +var z9719 [1 << 17]byte +var z9720 [1 << 17]byte +var z9721 [1 << 17]byte +var z9722 [1 << 17]byte +var z9723 [1 << 17]byte +var z9724 [1 << 17]byte +var z9725 [1 << 17]byte +var z9726 [1 << 17]byte +var z9727 [1 << 17]byte +var z9728 [1 << 17]byte +var z9729 [1 << 17]byte +var z9730 [1 << 17]byte +var z9731 [1 << 17]byte +var z9732 [1 << 17]byte +var z9733 [1 << 17]byte +var z9734 [1 << 17]byte +var z9735 [1 << 17]byte +var z9736 [1 << 17]byte +var z9737 [1 << 17]byte +var z9738 [1 << 17]byte +var z9739 [1 << 17]byte +var z9740 [1 << 17]byte +var z9741 [1 << 17]byte +var z9742 [1 << 17]byte +var z9743 [1 << 17]byte +var z9744 [1 << 17]byte +var z9745 [1 << 17]byte +var z9746 [1 << 17]byte +var z9747 [1 << 17]byte +var z9748 [1 << 17]byte +var z9749 [1 << 17]byte +var z9750 [1 << 17]byte +var z9751 [1 << 17]byte +var z9752 [1 << 17]byte +var z9753 [1 << 17]byte +var z9754 [1 << 17]byte +var z9755 [1 << 17]byte +var z9756 [1 << 17]byte +var z9757 [1 << 17]byte +var z9758 [1 << 17]byte +var z9759 [1 << 17]byte +var z9760 [1 << 17]byte +var z9761 [1 << 17]byte +var z9762 [1 << 17]byte +var z9763 [1 << 17]byte +var z9764 [1 << 17]byte +var z9765 [1 << 17]byte +var z9766 [1 << 17]byte +var z9767 [1 << 17]byte +var z9768 [1 << 17]byte +var z9769 [1 << 17]byte +var z9770 [1 << 17]byte +var z9771 [1 << 17]byte +var z9772 [1 << 17]byte +var z9773 [1 << 17]byte +var z9774 [1 << 17]byte +var z9775 [1 << 17]byte +var z9776 [1 << 17]byte +var z9777 [1 << 17]byte +var z9778 [1 << 17]byte +var z9779 [1 << 17]byte +var z9780 [1 << 17]byte +var z9781 [1 << 17]byte +var z9782 [1 << 17]byte +var z9783 [1 << 17]byte +var z9784 [1 << 17]byte +var z9785 [1 << 17]byte +var z9786 [1 << 17]byte +var z9787 [1 << 17]byte +var z9788 [1 << 17]byte +var z9789 [1 << 17]byte +var z9790 [1 << 17]byte +var z9791 [1 << 17]byte +var z9792 [1 << 17]byte +var z9793 [1 << 17]byte +var z9794 [1 << 17]byte +var z9795 [1 << 17]byte +var z9796 [1 << 17]byte +var z9797 [1 << 17]byte +var z9798 [1 << 17]byte +var z9799 [1 << 17]byte +var z9800 [1 << 17]byte +var z9801 [1 << 17]byte +var z9802 [1 << 17]byte +var z9803 [1 << 17]byte +var z9804 [1 << 17]byte +var z9805 [1 << 17]byte +var z9806 [1 << 17]byte +var z9807 [1 << 17]byte +var z9808 [1 << 17]byte +var z9809 [1 << 17]byte +var z9810 [1 << 17]byte +var z9811 [1 << 17]byte +var z9812 [1 << 17]byte +var z9813 [1 << 17]byte +var z9814 [1 << 17]byte +var z9815 [1 << 17]byte +var z9816 [1 << 17]byte +var z9817 [1 << 17]byte +var z9818 [1 << 17]byte +var z9819 [1 << 17]byte +var z9820 [1 << 17]byte +var z9821 [1 << 17]byte +var z9822 [1 << 17]byte +var z9823 [1 << 17]byte +var z9824 [1 << 17]byte +var z9825 [1 << 17]byte +var z9826 [1 << 17]byte +var z9827 [1 << 17]byte +var z9828 [1 << 17]byte +var z9829 [1 << 17]byte +var z9830 [1 << 17]byte +var z9831 [1 << 17]byte +var z9832 [1 << 17]byte +var z9833 [1 << 17]byte +var z9834 [1 << 17]byte +var z9835 [1 << 17]byte +var z9836 [1 << 17]byte +var z9837 [1 << 17]byte +var z9838 [1 << 17]byte +var z9839 [1 << 17]byte +var z9840 [1 << 17]byte +var z9841 [1 << 17]byte +var z9842 [1 << 17]byte +var z9843 [1 << 17]byte +var z9844 [1 << 17]byte +var z9845 [1 << 17]byte +var z9846 [1 << 17]byte +var z9847 [1 << 17]byte +var z9848 [1 << 17]byte +var z9849 [1 << 17]byte +var z9850 [1 << 17]byte +var z9851 [1 << 17]byte +var z9852 [1 << 17]byte +var z9853 [1 << 17]byte +var z9854 [1 << 17]byte +var z9855 [1 << 17]byte +var z9856 [1 << 17]byte +var z9857 [1 << 17]byte +var z9858 [1 << 17]byte +var z9859 [1 << 17]byte +var z9860 [1 << 17]byte +var z9861 [1 << 17]byte +var z9862 [1 << 17]byte +var z9863 [1 << 17]byte +var z9864 [1 << 17]byte +var z9865 [1 << 17]byte +var z9866 [1 << 17]byte +var z9867 [1 << 17]byte +var z9868 [1 << 17]byte +var z9869 [1 << 17]byte +var z9870 [1 << 17]byte +var z9871 [1 << 17]byte +var z9872 [1 << 17]byte +var z9873 [1 << 17]byte +var z9874 [1 << 17]byte +var z9875 [1 << 17]byte +var z9876 [1 << 17]byte +var z9877 [1 << 17]byte +var z9878 [1 << 17]byte +var z9879 [1 << 17]byte +var z9880 [1 << 17]byte +var z9881 [1 << 17]byte +var z9882 [1 << 17]byte +var z9883 [1 << 17]byte +var z9884 [1 << 17]byte +var z9885 [1 << 17]byte +var z9886 [1 << 17]byte +var z9887 [1 << 17]byte +var z9888 [1 << 17]byte +var z9889 [1 << 17]byte +var z9890 [1 << 17]byte +var z9891 [1 << 17]byte +var z9892 [1 << 17]byte +var z9893 [1 << 17]byte +var z9894 [1 << 17]byte +var z9895 [1 << 17]byte +var z9896 [1 << 17]byte +var z9897 [1 << 17]byte +var z9898 [1 << 17]byte +var z9899 [1 << 17]byte +var z9900 [1 << 17]byte +var z9901 [1 << 17]byte +var z9902 [1 << 17]byte +var z9903 [1 << 17]byte +var z9904 [1 << 17]byte +var z9905 [1 << 17]byte +var z9906 [1 << 17]byte +var z9907 [1 << 17]byte +var z9908 [1 << 17]byte +var z9909 [1 << 17]byte +var z9910 [1 << 17]byte +var z9911 [1 << 17]byte +var z9912 [1 << 17]byte +var z9913 [1 << 17]byte +var z9914 [1 << 17]byte +var z9915 [1 << 17]byte +var z9916 [1 << 17]byte +var z9917 [1 << 17]byte +var z9918 [1 << 17]byte +var z9919 [1 << 17]byte +var z9920 [1 << 17]byte +var z9921 [1 << 17]byte +var z9922 [1 << 17]byte +var z9923 [1 << 17]byte +var z9924 [1 << 17]byte +var z9925 [1 << 17]byte +var z9926 [1 << 17]byte +var z9927 [1 << 17]byte +var z9928 [1 << 17]byte +var z9929 [1 << 17]byte +var z9930 [1 << 17]byte +var z9931 [1 << 17]byte +var z9932 [1 << 17]byte +var z9933 [1 << 17]byte +var z9934 [1 << 17]byte +var z9935 [1 << 17]byte +var z9936 [1 << 17]byte +var z9937 [1 << 17]byte +var z9938 [1 << 17]byte +var z9939 [1 << 17]byte +var z9940 [1 << 17]byte +var z9941 [1 << 17]byte +var z9942 [1 << 17]byte +var z9943 [1 << 17]byte +var z9944 [1 << 17]byte +var z9945 [1 << 17]byte +var z9946 [1 << 17]byte +var z9947 [1 << 17]byte +var z9948 [1 << 17]byte +var z9949 [1 << 17]byte +var z9950 [1 << 17]byte +var z9951 [1 << 17]byte +var z9952 [1 << 17]byte +var z9953 [1 << 17]byte +var z9954 [1 << 17]byte +var z9955 [1 << 17]byte +var z9956 [1 << 17]byte +var z9957 [1 << 17]byte +var z9958 [1 << 17]byte +var z9959 [1 << 17]byte +var z9960 [1 << 17]byte +var z9961 [1 << 17]byte +var z9962 [1 << 17]byte +var z9963 [1 << 17]byte +var z9964 [1 << 17]byte +var z9965 [1 << 17]byte +var z9966 [1 << 17]byte +var z9967 [1 << 17]byte +var z9968 [1 << 17]byte +var z9969 [1 << 17]byte +var z9970 [1 << 17]byte +var z9971 [1 << 17]byte +var z9972 [1 << 17]byte +var z9973 [1 << 17]byte +var z9974 [1 << 17]byte +var z9975 [1 << 17]byte +var z9976 [1 << 17]byte +var z9977 [1 << 17]byte +var z9978 [1 << 17]byte +var z9979 [1 << 17]byte +var z9980 [1 << 17]byte +var z9981 [1 << 17]byte +var z9982 [1 << 17]byte +var z9983 [1 << 17]byte +var z9984 [1 << 17]byte +var z9985 [1 << 17]byte +var z9986 [1 << 17]byte +var z9987 [1 << 17]byte +var z9988 [1 << 17]byte +var z9989 [1 << 17]byte +var z9990 [1 << 17]byte +var z9991 [1 << 17]byte +var z9992 [1 << 17]byte +var z9993 [1 << 17]byte +var z9994 [1 << 17]byte +var z9995 [1 << 17]byte +var z9996 [1 << 17]byte +var z9997 [1 << 17]byte +var z9998 [1 << 17]byte +var z9999 [1 << 17]byte +var z10000 [1 << 17]byte +var z10001 [1 << 17]byte +var z10002 [1 << 17]byte +var z10003 [1 << 17]byte +var z10004 [1 << 17]byte +var z10005 [1 << 17]byte +var z10006 [1 << 17]byte +var z10007 [1 << 17]byte +var z10008 [1 << 17]byte +var z10009 [1 << 17]byte +var z10010 [1 << 17]byte +var z10011 [1 << 17]byte +var z10012 [1 << 17]byte +var z10013 [1 << 17]byte +var z10014 [1 << 17]byte +var z10015 [1 << 17]byte +var z10016 [1 << 17]byte +var z10017 [1 << 17]byte +var z10018 [1 << 17]byte +var z10019 [1 << 17]byte +var z10020 [1 << 17]byte +var z10021 [1 << 17]byte +var z10022 [1 << 17]byte +var z10023 [1 << 17]byte +var z10024 [1 << 17]byte +var z10025 [1 << 17]byte +var z10026 [1 << 17]byte +var z10027 [1 << 17]byte +var z10028 [1 << 17]byte +var z10029 [1 << 17]byte +var z10030 [1 << 17]byte +var z10031 [1 << 17]byte +var z10032 [1 << 17]byte +var z10033 [1 << 17]byte +var z10034 [1 << 17]byte +var z10035 [1 << 17]byte +var z10036 [1 << 17]byte +var z10037 [1 << 17]byte +var z10038 [1 << 17]byte +var z10039 [1 << 17]byte +var z10040 [1 << 17]byte +var z10041 [1 << 17]byte +var z10042 [1 << 17]byte +var z10043 [1 << 17]byte +var z10044 [1 << 17]byte +var z10045 [1 << 17]byte +var z10046 [1 << 17]byte +var z10047 [1 << 17]byte +var z10048 [1 << 17]byte +var z10049 [1 << 17]byte +var z10050 [1 << 17]byte +var z10051 [1 << 17]byte +var z10052 [1 << 17]byte +var z10053 [1 << 17]byte +var z10054 [1 << 17]byte +var z10055 [1 << 17]byte +var z10056 [1 << 17]byte +var z10057 [1 << 17]byte +var z10058 [1 << 17]byte +var z10059 [1 << 17]byte +var z10060 [1 << 17]byte +var z10061 [1 << 17]byte +var z10062 [1 << 17]byte +var z10063 [1 << 17]byte +var z10064 [1 << 17]byte +var z10065 [1 << 17]byte +var z10066 [1 << 17]byte +var z10067 [1 << 17]byte +var z10068 [1 << 17]byte +var z10069 [1 << 17]byte +var z10070 [1 << 17]byte +var z10071 [1 << 17]byte +var z10072 [1 << 17]byte +var z10073 [1 << 17]byte +var z10074 [1 << 17]byte +var z10075 [1 << 17]byte +var z10076 [1 << 17]byte +var z10077 [1 << 17]byte +var z10078 [1 << 17]byte +var z10079 [1 << 17]byte +var z10080 [1 << 17]byte +var z10081 [1 << 17]byte +var z10082 [1 << 17]byte +var z10083 [1 << 17]byte +var z10084 [1 << 17]byte +var z10085 [1 << 17]byte +var z10086 [1 << 17]byte +var z10087 [1 << 17]byte +var z10088 [1 << 17]byte +var z10089 [1 << 17]byte +var z10090 [1 << 17]byte +var z10091 [1 << 17]byte +var z10092 [1 << 17]byte +var z10093 [1 << 17]byte +var z10094 [1 << 17]byte +var z10095 [1 << 17]byte +var z10096 [1 << 17]byte +var z10097 [1 << 17]byte +var z10098 [1 << 17]byte +var z10099 [1 << 17]byte +var z10100 [1 << 17]byte +var z10101 [1 << 17]byte +var z10102 [1 << 17]byte +var z10103 [1 << 17]byte +var z10104 [1 << 17]byte +var z10105 [1 << 17]byte +var z10106 [1 << 17]byte +var z10107 [1 << 17]byte +var z10108 [1 << 17]byte +var z10109 [1 << 17]byte +var z10110 [1 << 17]byte +var z10111 [1 << 17]byte +var z10112 [1 << 17]byte +var z10113 [1 << 17]byte +var z10114 [1 << 17]byte +var z10115 [1 << 17]byte +var z10116 [1 << 17]byte +var z10117 [1 << 17]byte +var z10118 [1 << 17]byte +var z10119 [1 << 17]byte +var z10120 [1 << 17]byte +var z10121 [1 << 17]byte +var z10122 [1 << 17]byte +var z10123 [1 << 17]byte +var z10124 [1 << 17]byte +var z10125 [1 << 17]byte +var z10126 [1 << 17]byte +var z10127 [1 << 17]byte +var z10128 [1 << 17]byte +var z10129 [1 << 17]byte +var z10130 [1 << 17]byte +var z10131 [1 << 17]byte +var z10132 [1 << 17]byte +var z10133 [1 << 17]byte +var z10134 [1 << 17]byte +var z10135 [1 << 17]byte +var z10136 [1 << 17]byte +var z10137 [1 << 17]byte +var z10138 [1 << 17]byte +var z10139 [1 << 17]byte +var z10140 [1 << 17]byte +var z10141 [1 << 17]byte +var z10142 [1 << 17]byte +var z10143 [1 << 17]byte +var z10144 [1 << 17]byte +var z10145 [1 << 17]byte +var z10146 [1 << 17]byte +var z10147 [1 << 17]byte +var z10148 [1 << 17]byte +var z10149 [1 << 17]byte +var z10150 [1 << 17]byte +var z10151 [1 << 17]byte +var z10152 [1 << 17]byte +var z10153 [1 << 17]byte +var z10154 [1 << 17]byte +var z10155 [1 << 17]byte +var z10156 [1 << 17]byte +var z10157 [1 << 17]byte +var z10158 [1 << 17]byte +var z10159 [1 << 17]byte +var z10160 [1 << 17]byte +var z10161 [1 << 17]byte +var z10162 [1 << 17]byte +var z10163 [1 << 17]byte +var z10164 [1 << 17]byte +var z10165 [1 << 17]byte +var z10166 [1 << 17]byte +var z10167 [1 << 17]byte +var z10168 [1 << 17]byte +var z10169 [1 << 17]byte +var z10170 [1 << 17]byte +var z10171 [1 << 17]byte +var z10172 [1 << 17]byte +var z10173 [1 << 17]byte +var z10174 [1 << 17]byte +var z10175 [1 << 17]byte +var z10176 [1 << 17]byte +var z10177 [1 << 17]byte +var z10178 [1 << 17]byte +var z10179 [1 << 17]byte +var z10180 [1 << 17]byte +var z10181 [1 << 17]byte +var z10182 [1 << 17]byte +var z10183 [1 << 17]byte +var z10184 [1 << 17]byte +var z10185 [1 << 17]byte +var z10186 [1 << 17]byte +var z10187 [1 << 17]byte +var z10188 [1 << 17]byte +var z10189 [1 << 17]byte +var z10190 [1 << 17]byte +var z10191 [1 << 17]byte +var z10192 [1 << 17]byte +var z10193 [1 << 17]byte +var z10194 [1 << 17]byte +var z10195 [1 << 17]byte +var z10196 [1 << 17]byte +var z10197 [1 << 17]byte +var z10198 [1 << 17]byte +var z10199 [1 << 17]byte +var z10200 [1 << 17]byte +var z10201 [1 << 17]byte +var z10202 [1 << 17]byte +var z10203 [1 << 17]byte +var z10204 [1 << 17]byte +var z10205 [1 << 17]byte +var z10206 [1 << 17]byte +var z10207 [1 << 17]byte +var z10208 [1 << 17]byte +var z10209 [1 << 17]byte +var z10210 [1 << 17]byte +var z10211 [1 << 17]byte +var z10212 [1 << 17]byte +var z10213 [1 << 17]byte +var z10214 [1 << 17]byte +var z10215 [1 << 17]byte +var z10216 [1 << 17]byte +var z10217 [1 << 17]byte +var z10218 [1 << 17]byte +var z10219 [1 << 17]byte +var z10220 [1 << 17]byte +var z10221 [1 << 17]byte +var z10222 [1 << 17]byte +var z10223 [1 << 17]byte +var z10224 [1 << 17]byte +var z10225 [1 << 17]byte +var z10226 [1 << 17]byte +var z10227 [1 << 17]byte +var z10228 [1 << 17]byte +var z10229 [1 << 17]byte +var z10230 [1 << 17]byte +var z10231 [1 << 17]byte +var z10232 [1 << 17]byte +var z10233 [1 << 17]byte +var z10234 [1 << 17]byte +var z10235 [1 << 17]byte +var z10236 [1 << 17]byte +var z10237 [1 << 17]byte +var z10238 [1 << 17]byte +var z10239 [1 << 17]byte +var z10240 [1 << 17]byte +var z10241 [1 << 17]byte +var z10242 [1 << 17]byte +var z10243 [1 << 17]byte +var z10244 [1 << 17]byte +var z10245 [1 << 17]byte +var z10246 [1 << 17]byte +var z10247 [1 << 17]byte +var z10248 [1 << 17]byte +var z10249 [1 << 17]byte +var z10250 [1 << 17]byte +var z10251 [1 << 17]byte +var z10252 [1 << 17]byte +var z10253 [1 << 17]byte +var z10254 [1 << 17]byte +var z10255 [1 << 17]byte +var z10256 [1 << 17]byte +var z10257 [1 << 17]byte +var z10258 [1 << 17]byte +var z10259 [1 << 17]byte +var z10260 [1 << 17]byte +var z10261 [1 << 17]byte +var z10262 [1 << 17]byte +var z10263 [1 << 17]byte +var z10264 [1 << 17]byte +var z10265 [1 << 17]byte +var z10266 [1 << 17]byte +var z10267 [1 << 17]byte +var z10268 [1 << 17]byte +var z10269 [1 << 17]byte +var z10270 [1 << 17]byte +var z10271 [1 << 17]byte +var z10272 [1 << 17]byte +var z10273 [1 << 17]byte +var z10274 [1 << 17]byte +var z10275 [1 << 17]byte +var z10276 [1 << 17]byte +var z10277 [1 << 17]byte +var z10278 [1 << 17]byte +var z10279 [1 << 17]byte +var z10280 [1 << 17]byte +var z10281 [1 << 17]byte +var z10282 [1 << 17]byte +var z10283 [1 << 17]byte +var z10284 [1 << 17]byte +var z10285 [1 << 17]byte +var z10286 [1 << 17]byte +var z10287 [1 << 17]byte +var z10288 [1 << 17]byte +var z10289 [1 << 17]byte +var z10290 [1 << 17]byte +var z10291 [1 << 17]byte +var z10292 [1 << 17]byte +var z10293 [1 << 17]byte +var z10294 [1 << 17]byte +var z10295 [1 << 17]byte +var z10296 [1 << 17]byte +var z10297 [1 << 17]byte +var z10298 [1 << 17]byte +var z10299 [1 << 17]byte +var z10300 [1 << 17]byte +var z10301 [1 << 17]byte +var z10302 [1 << 17]byte +var z10303 [1 << 17]byte +var z10304 [1 << 17]byte +var z10305 [1 << 17]byte +var z10306 [1 << 17]byte +var z10307 [1 << 17]byte +var z10308 [1 << 17]byte +var z10309 [1 << 17]byte +var z10310 [1 << 17]byte +var z10311 [1 << 17]byte +var z10312 [1 << 17]byte +var z10313 [1 << 17]byte +var z10314 [1 << 17]byte +var z10315 [1 << 17]byte +var z10316 [1 << 17]byte +var z10317 [1 << 17]byte +var z10318 [1 << 17]byte +var z10319 [1 << 17]byte +var z10320 [1 << 17]byte +var z10321 [1 << 17]byte +var z10322 [1 << 17]byte +var z10323 [1 << 17]byte +var z10324 [1 << 17]byte +var z10325 [1 << 17]byte +var z10326 [1 << 17]byte +var z10327 [1 << 17]byte +var z10328 [1 << 17]byte +var z10329 [1 << 17]byte +var z10330 [1 << 17]byte +var z10331 [1 << 17]byte +var z10332 [1 << 17]byte +var z10333 [1 << 17]byte +var z10334 [1 << 17]byte +var z10335 [1 << 17]byte +var z10336 [1 << 17]byte +var z10337 [1 << 17]byte +var z10338 [1 << 17]byte +var z10339 [1 << 17]byte +var z10340 [1 << 17]byte +var z10341 [1 << 17]byte +var z10342 [1 << 17]byte +var z10343 [1 << 17]byte +var z10344 [1 << 17]byte +var z10345 [1 << 17]byte +var z10346 [1 << 17]byte +var z10347 [1 << 17]byte +var z10348 [1 << 17]byte +var z10349 [1 << 17]byte +var z10350 [1 << 17]byte +var z10351 [1 << 17]byte +var z10352 [1 << 17]byte +var z10353 [1 << 17]byte +var z10354 [1 << 17]byte +var z10355 [1 << 17]byte +var z10356 [1 << 17]byte +var z10357 [1 << 17]byte +var z10358 [1 << 17]byte +var z10359 [1 << 17]byte +var z10360 [1 << 17]byte +var z10361 [1 << 17]byte +var z10362 [1 << 17]byte +var z10363 [1 << 17]byte +var z10364 [1 << 17]byte +var z10365 [1 << 17]byte +var z10366 [1 << 17]byte +var z10367 [1 << 17]byte +var z10368 [1 << 17]byte +var z10369 [1 << 17]byte +var z10370 [1 << 17]byte +var z10371 [1 << 17]byte +var z10372 [1 << 17]byte +var z10373 [1 << 17]byte +var z10374 [1 << 17]byte +var z10375 [1 << 17]byte +var z10376 [1 << 17]byte +var z10377 [1 << 17]byte +var z10378 [1 << 17]byte +var z10379 [1 << 17]byte +var z10380 [1 << 17]byte +var z10381 [1 << 17]byte +var z10382 [1 << 17]byte +var z10383 [1 << 17]byte +var z10384 [1 << 17]byte +var z10385 [1 << 17]byte +var z10386 [1 << 17]byte +var z10387 [1 << 17]byte +var z10388 [1 << 17]byte +var z10389 [1 << 17]byte +var z10390 [1 << 17]byte +var z10391 [1 << 17]byte +var z10392 [1 << 17]byte +var z10393 [1 << 17]byte +var z10394 [1 << 17]byte +var z10395 [1 << 17]byte +var z10396 [1 << 17]byte +var z10397 [1 << 17]byte +var z10398 [1 << 17]byte +var z10399 [1 << 17]byte +var z10400 [1 << 17]byte +var z10401 [1 << 17]byte +var z10402 [1 << 17]byte +var z10403 [1 << 17]byte +var z10404 [1 << 17]byte +var z10405 [1 << 17]byte +var z10406 [1 << 17]byte +var z10407 [1 << 17]byte +var z10408 [1 << 17]byte +var z10409 [1 << 17]byte +var z10410 [1 << 17]byte +var z10411 [1 << 17]byte +var z10412 [1 << 17]byte +var z10413 [1 << 17]byte +var z10414 [1 << 17]byte +var z10415 [1 << 17]byte +var z10416 [1 << 17]byte +var z10417 [1 << 17]byte +var z10418 [1 << 17]byte +var z10419 [1 << 17]byte +var z10420 [1 << 17]byte +var z10421 [1 << 17]byte +var z10422 [1 << 17]byte +var z10423 [1 << 17]byte +var z10424 [1 << 17]byte +var z10425 [1 << 17]byte +var z10426 [1 << 17]byte +var z10427 [1 << 17]byte +var z10428 [1 << 17]byte +var z10429 [1 << 17]byte +var z10430 [1 << 17]byte +var z10431 [1 << 17]byte +var z10432 [1 << 17]byte +var z10433 [1 << 17]byte +var z10434 [1 << 17]byte +var z10435 [1 << 17]byte +var z10436 [1 << 17]byte +var z10437 [1 << 17]byte +var z10438 [1 << 17]byte +var z10439 [1 << 17]byte +var z10440 [1 << 17]byte +var z10441 [1 << 17]byte +var z10442 [1 << 17]byte +var z10443 [1 << 17]byte +var z10444 [1 << 17]byte +var z10445 [1 << 17]byte +var z10446 [1 << 17]byte +var z10447 [1 << 17]byte +var z10448 [1 << 17]byte +var z10449 [1 << 17]byte +var z10450 [1 << 17]byte +var z10451 [1 << 17]byte +var z10452 [1 << 17]byte +var z10453 [1 << 17]byte +var z10454 [1 << 17]byte +var z10455 [1 << 17]byte +var z10456 [1 << 17]byte +var z10457 [1 << 17]byte +var z10458 [1 << 17]byte +var z10459 [1 << 17]byte +var z10460 [1 << 17]byte +var z10461 [1 << 17]byte +var z10462 [1 << 17]byte +var z10463 [1 << 17]byte +var z10464 [1 << 17]byte +var z10465 [1 << 17]byte +var z10466 [1 << 17]byte +var z10467 [1 << 17]byte +var z10468 [1 << 17]byte +var z10469 [1 << 17]byte +var z10470 [1 << 17]byte +var z10471 [1 << 17]byte +var z10472 [1 << 17]byte +var z10473 [1 << 17]byte +var z10474 [1 << 17]byte +var z10475 [1 << 17]byte +var z10476 [1 << 17]byte +var z10477 [1 << 17]byte +var z10478 [1 << 17]byte +var z10479 [1 << 17]byte +var z10480 [1 << 17]byte +var z10481 [1 << 17]byte +var z10482 [1 << 17]byte +var z10483 [1 << 17]byte +var z10484 [1 << 17]byte +var z10485 [1 << 17]byte +var z10486 [1 << 17]byte +var z10487 [1 << 17]byte +var z10488 [1 << 17]byte +var z10489 [1 << 17]byte +var z10490 [1 << 17]byte +var z10491 [1 << 17]byte +var z10492 [1 << 17]byte +var z10493 [1 << 17]byte +var z10494 [1 << 17]byte +var z10495 [1 << 17]byte +var z10496 [1 << 17]byte +var z10497 [1 << 17]byte +var z10498 [1 << 17]byte +var z10499 [1 << 17]byte +var z10500 [1 << 17]byte +var z10501 [1 << 17]byte +var z10502 [1 << 17]byte +var z10503 [1 << 17]byte +var z10504 [1 << 17]byte +var z10505 [1 << 17]byte +var z10506 [1 << 17]byte +var z10507 [1 << 17]byte +var z10508 [1 << 17]byte +var z10509 [1 << 17]byte +var z10510 [1 << 17]byte +var z10511 [1 << 17]byte +var z10512 [1 << 17]byte +var z10513 [1 << 17]byte +var z10514 [1 << 17]byte +var z10515 [1 << 17]byte +var z10516 [1 << 17]byte +var z10517 [1 << 17]byte +var z10518 [1 << 17]byte +var z10519 [1 << 17]byte +var z10520 [1 << 17]byte +var z10521 [1 << 17]byte +var z10522 [1 << 17]byte +var z10523 [1 << 17]byte +var z10524 [1 << 17]byte +var z10525 [1 << 17]byte +var z10526 [1 << 17]byte +var z10527 [1 << 17]byte +var z10528 [1 << 17]byte +var z10529 [1 << 17]byte +var z10530 [1 << 17]byte +var z10531 [1 << 17]byte +var z10532 [1 << 17]byte +var z10533 [1 << 17]byte +var z10534 [1 << 17]byte +var z10535 [1 << 17]byte +var z10536 [1 << 17]byte +var z10537 [1 << 17]byte +var z10538 [1 << 17]byte +var z10539 [1 << 17]byte +var z10540 [1 << 17]byte +var z10541 [1 << 17]byte +var z10542 [1 << 17]byte +var z10543 [1 << 17]byte +var z10544 [1 << 17]byte +var z10545 [1 << 17]byte +var z10546 [1 << 17]byte +var z10547 [1 << 17]byte +var z10548 [1 << 17]byte +var z10549 [1 << 17]byte +var z10550 [1 << 17]byte +var z10551 [1 << 17]byte +var z10552 [1 << 17]byte +var z10553 [1 << 17]byte +var z10554 [1 << 17]byte +var z10555 [1 << 17]byte +var z10556 [1 << 17]byte +var z10557 [1 << 17]byte +var z10558 [1 << 17]byte +var z10559 [1 << 17]byte +var z10560 [1 << 17]byte +var z10561 [1 << 17]byte +var z10562 [1 << 17]byte +var z10563 [1 << 17]byte +var z10564 [1 << 17]byte +var z10565 [1 << 17]byte +var z10566 [1 << 17]byte +var z10567 [1 << 17]byte +var z10568 [1 << 17]byte +var z10569 [1 << 17]byte +var z10570 [1 << 17]byte +var z10571 [1 << 17]byte +var z10572 [1 << 17]byte +var z10573 [1 << 17]byte +var z10574 [1 << 17]byte +var z10575 [1 << 17]byte +var z10576 [1 << 17]byte +var z10577 [1 << 17]byte +var z10578 [1 << 17]byte +var z10579 [1 << 17]byte +var z10580 [1 << 17]byte +var z10581 [1 << 17]byte +var z10582 [1 << 17]byte +var z10583 [1 << 17]byte +var z10584 [1 << 17]byte +var z10585 [1 << 17]byte +var z10586 [1 << 17]byte +var z10587 [1 << 17]byte +var z10588 [1 << 17]byte +var z10589 [1 << 17]byte +var z10590 [1 << 17]byte +var z10591 [1 << 17]byte +var z10592 [1 << 17]byte +var z10593 [1 << 17]byte +var z10594 [1 << 17]byte +var z10595 [1 << 17]byte +var z10596 [1 << 17]byte +var z10597 [1 << 17]byte +var z10598 [1 << 17]byte +var z10599 [1 << 17]byte +var z10600 [1 << 17]byte +var z10601 [1 << 17]byte +var z10602 [1 << 17]byte +var z10603 [1 << 17]byte +var z10604 [1 << 17]byte +var z10605 [1 << 17]byte +var z10606 [1 << 17]byte +var z10607 [1 << 17]byte +var z10608 [1 << 17]byte +var z10609 [1 << 17]byte +var z10610 [1 << 17]byte +var z10611 [1 << 17]byte +var z10612 [1 << 17]byte +var z10613 [1 << 17]byte +var z10614 [1 << 17]byte +var z10615 [1 << 17]byte +var z10616 [1 << 17]byte +var z10617 [1 << 17]byte +var z10618 [1 << 17]byte +var z10619 [1 << 17]byte +var z10620 [1 << 17]byte +var z10621 [1 << 17]byte +var z10622 [1 << 17]byte +var z10623 [1 << 17]byte +var z10624 [1 << 17]byte +var z10625 [1 << 17]byte +var z10626 [1 << 17]byte +var z10627 [1 << 17]byte +var z10628 [1 << 17]byte +var z10629 [1 << 17]byte +var z10630 [1 << 17]byte +var z10631 [1 << 17]byte +var z10632 [1 << 17]byte +var z10633 [1 << 17]byte +var z10634 [1 << 17]byte +var z10635 [1 << 17]byte +var z10636 [1 << 17]byte +var z10637 [1 << 17]byte +var z10638 [1 << 17]byte +var z10639 [1 << 17]byte +var z10640 [1 << 17]byte +var z10641 [1 << 17]byte +var z10642 [1 << 17]byte +var z10643 [1 << 17]byte +var z10644 [1 << 17]byte +var z10645 [1 << 17]byte +var z10646 [1 << 17]byte +var z10647 [1 << 17]byte +var z10648 [1 << 17]byte +var z10649 [1 << 17]byte +var z10650 [1 << 17]byte +var z10651 [1 << 17]byte +var z10652 [1 << 17]byte +var z10653 [1 << 17]byte +var z10654 [1 << 17]byte +var z10655 [1 << 17]byte +var z10656 [1 << 17]byte +var z10657 [1 << 17]byte +var z10658 [1 << 17]byte +var z10659 [1 << 17]byte +var z10660 [1 << 17]byte +var z10661 [1 << 17]byte +var z10662 [1 << 17]byte +var z10663 [1 << 17]byte +var z10664 [1 << 17]byte +var z10665 [1 << 17]byte +var z10666 [1 << 17]byte +var z10667 [1 << 17]byte +var z10668 [1 << 17]byte +var z10669 [1 << 17]byte +var z10670 [1 << 17]byte +var z10671 [1 << 17]byte +var z10672 [1 << 17]byte +var z10673 [1 << 17]byte +var z10674 [1 << 17]byte +var z10675 [1 << 17]byte +var z10676 [1 << 17]byte +var z10677 [1 << 17]byte +var z10678 [1 << 17]byte +var z10679 [1 << 17]byte +var z10680 [1 << 17]byte +var z10681 [1 << 17]byte +var z10682 [1 << 17]byte +var z10683 [1 << 17]byte +var z10684 [1 << 17]byte +var z10685 [1 << 17]byte +var z10686 [1 << 17]byte +var z10687 [1 << 17]byte +var z10688 [1 << 17]byte +var z10689 [1 << 17]byte +var z10690 [1 << 17]byte +var z10691 [1 << 17]byte +var z10692 [1 << 17]byte +var z10693 [1 << 17]byte +var z10694 [1 << 17]byte +var z10695 [1 << 17]byte +var z10696 [1 << 17]byte +var z10697 [1 << 17]byte +var z10698 [1 << 17]byte +var z10699 [1 << 17]byte +var z10700 [1 << 17]byte +var z10701 [1 << 17]byte +var z10702 [1 << 17]byte +var z10703 [1 << 17]byte +var z10704 [1 << 17]byte +var z10705 [1 << 17]byte +var z10706 [1 << 17]byte +var z10707 [1 << 17]byte +var z10708 [1 << 17]byte +var z10709 [1 << 17]byte +var z10710 [1 << 17]byte +var z10711 [1 << 17]byte +var z10712 [1 << 17]byte +var z10713 [1 << 17]byte +var z10714 [1 << 17]byte +var z10715 [1 << 17]byte +var z10716 [1 << 17]byte +var z10717 [1 << 17]byte +var z10718 [1 << 17]byte +var z10719 [1 << 17]byte +var z10720 [1 << 17]byte +var z10721 [1 << 17]byte +var z10722 [1 << 17]byte +var z10723 [1 << 17]byte +var z10724 [1 << 17]byte +var z10725 [1 << 17]byte +var z10726 [1 << 17]byte +var z10727 [1 << 17]byte +var z10728 [1 << 17]byte +var z10729 [1 << 17]byte +var z10730 [1 << 17]byte +var z10731 [1 << 17]byte +var z10732 [1 << 17]byte +var z10733 [1 << 17]byte +var z10734 [1 << 17]byte +var z10735 [1 << 17]byte +var z10736 [1 << 17]byte +var z10737 [1 << 17]byte +var z10738 [1 << 17]byte +var z10739 [1 << 17]byte +var z10740 [1 << 17]byte +var z10741 [1 << 17]byte +var z10742 [1 << 17]byte +var z10743 [1 << 17]byte +var z10744 [1 << 17]byte +var z10745 [1 << 17]byte +var z10746 [1 << 17]byte +var z10747 [1 << 17]byte +var z10748 [1 << 17]byte +var z10749 [1 << 17]byte +var z10750 [1 << 17]byte +var z10751 [1 << 17]byte +var z10752 [1 << 17]byte +var z10753 [1 << 17]byte +var z10754 [1 << 17]byte +var z10755 [1 << 17]byte +var z10756 [1 << 17]byte +var z10757 [1 << 17]byte +var z10758 [1 << 17]byte +var z10759 [1 << 17]byte +var z10760 [1 << 17]byte +var z10761 [1 << 17]byte +var z10762 [1 << 17]byte +var z10763 [1 << 17]byte +var z10764 [1 << 17]byte +var z10765 [1 << 17]byte +var z10766 [1 << 17]byte +var z10767 [1 << 17]byte +var z10768 [1 << 17]byte +var z10769 [1 << 17]byte +var z10770 [1 << 17]byte +var z10771 [1 << 17]byte +var z10772 [1 << 17]byte +var z10773 [1 << 17]byte +var z10774 [1 << 17]byte +var z10775 [1 << 17]byte +var z10776 [1 << 17]byte +var z10777 [1 << 17]byte +var z10778 [1 << 17]byte +var z10779 [1 << 17]byte +var z10780 [1 << 17]byte +var z10781 [1 << 17]byte +var z10782 [1 << 17]byte +var z10783 [1 << 17]byte +var z10784 [1 << 17]byte +var z10785 [1 << 17]byte +var z10786 [1 << 17]byte +var z10787 [1 << 17]byte +var z10788 [1 << 17]byte +var z10789 [1 << 17]byte +var z10790 [1 << 17]byte +var z10791 [1 << 17]byte +var z10792 [1 << 17]byte +var z10793 [1 << 17]byte +var z10794 [1 << 17]byte +var z10795 [1 << 17]byte +var z10796 [1 << 17]byte +var z10797 [1 << 17]byte +var z10798 [1 << 17]byte +var z10799 [1 << 17]byte +var z10800 [1 << 17]byte +var z10801 [1 << 17]byte +var z10802 [1 << 17]byte +var z10803 [1 << 17]byte +var z10804 [1 << 17]byte +var z10805 [1 << 17]byte +var z10806 [1 << 17]byte +var z10807 [1 << 17]byte +var z10808 [1 << 17]byte +var z10809 [1 << 17]byte +var z10810 [1 << 17]byte +var z10811 [1 << 17]byte +var z10812 [1 << 17]byte +var z10813 [1 << 17]byte +var z10814 [1 << 17]byte +var z10815 [1 << 17]byte +var z10816 [1 << 17]byte +var z10817 [1 << 17]byte +var z10818 [1 << 17]byte +var z10819 [1 << 17]byte +var z10820 [1 << 17]byte +var z10821 [1 << 17]byte +var z10822 [1 << 17]byte +var z10823 [1 << 17]byte +var z10824 [1 << 17]byte +var z10825 [1 << 17]byte +var z10826 [1 << 17]byte +var z10827 [1 << 17]byte +var z10828 [1 << 17]byte +var z10829 [1 << 17]byte +var z10830 [1 << 17]byte +var z10831 [1 << 17]byte +var z10832 [1 << 17]byte +var z10833 [1 << 17]byte +var z10834 [1 << 17]byte +var z10835 [1 << 17]byte +var z10836 [1 << 17]byte +var z10837 [1 << 17]byte +var z10838 [1 << 17]byte +var z10839 [1 << 17]byte +var z10840 [1 << 17]byte +var z10841 [1 << 17]byte +var z10842 [1 << 17]byte +var z10843 [1 << 17]byte +var z10844 [1 << 17]byte +var z10845 [1 << 17]byte +var z10846 [1 << 17]byte +var z10847 [1 << 17]byte +var z10848 [1 << 17]byte +var z10849 [1 << 17]byte +var z10850 [1 << 17]byte +var z10851 [1 << 17]byte +var z10852 [1 << 17]byte +var z10853 [1 << 17]byte +var z10854 [1 << 17]byte +var z10855 [1 << 17]byte +var z10856 [1 << 17]byte +var z10857 [1 << 17]byte +var z10858 [1 << 17]byte +var z10859 [1 << 17]byte +var z10860 [1 << 17]byte +var z10861 [1 << 17]byte +var z10862 [1 << 17]byte +var z10863 [1 << 17]byte +var z10864 [1 << 17]byte +var z10865 [1 << 17]byte +var z10866 [1 << 17]byte +var z10867 [1 << 17]byte +var z10868 [1 << 17]byte +var z10869 [1 << 17]byte +var z10870 [1 << 17]byte +var z10871 [1 << 17]byte +var z10872 [1 << 17]byte +var z10873 [1 << 17]byte +var z10874 [1 << 17]byte +var z10875 [1 << 17]byte +var z10876 [1 << 17]byte +var z10877 [1 << 17]byte +var z10878 [1 << 17]byte +var z10879 [1 << 17]byte +var z10880 [1 << 17]byte +var z10881 [1 << 17]byte +var z10882 [1 << 17]byte +var z10883 [1 << 17]byte +var z10884 [1 << 17]byte +var z10885 [1 << 17]byte +var z10886 [1 << 17]byte +var z10887 [1 << 17]byte +var z10888 [1 << 17]byte +var z10889 [1 << 17]byte +var z10890 [1 << 17]byte +var z10891 [1 << 17]byte +var z10892 [1 << 17]byte +var z10893 [1 << 17]byte +var z10894 [1 << 17]byte +var z10895 [1 << 17]byte +var z10896 [1 << 17]byte +var z10897 [1 << 17]byte +var z10898 [1 << 17]byte +var z10899 [1 << 17]byte +var z10900 [1 << 17]byte +var z10901 [1 << 17]byte +var z10902 [1 << 17]byte +var z10903 [1 << 17]byte +var z10904 [1 << 17]byte +var z10905 [1 << 17]byte +var z10906 [1 << 17]byte +var z10907 [1 << 17]byte +var z10908 [1 << 17]byte +var z10909 [1 << 17]byte +var z10910 [1 << 17]byte +var z10911 [1 << 17]byte +var z10912 [1 << 17]byte +var z10913 [1 << 17]byte +var z10914 [1 << 17]byte +var z10915 [1 << 17]byte +var z10916 [1 << 17]byte +var z10917 [1 << 17]byte +var z10918 [1 << 17]byte +var z10919 [1 << 17]byte +var z10920 [1 << 17]byte +var z10921 [1 << 17]byte +var z10922 [1 << 17]byte +var z10923 [1 << 17]byte +var z10924 [1 << 17]byte +var z10925 [1 << 17]byte +var z10926 [1 << 17]byte +var z10927 [1 << 17]byte +var z10928 [1 << 17]byte +var z10929 [1 << 17]byte +var z10930 [1 << 17]byte +var z10931 [1 << 17]byte +var z10932 [1 << 17]byte +var z10933 [1 << 17]byte +var z10934 [1 << 17]byte +var z10935 [1 << 17]byte +var z10936 [1 << 17]byte +var z10937 [1 << 17]byte +var z10938 [1 << 17]byte +var z10939 [1 << 17]byte +var z10940 [1 << 17]byte +var z10941 [1 << 17]byte +var z10942 [1 << 17]byte +var z10943 [1 << 17]byte +var z10944 [1 << 17]byte +var z10945 [1 << 17]byte +var z10946 [1 << 17]byte +var z10947 [1 << 17]byte +var z10948 [1 << 17]byte +var z10949 [1 << 17]byte +var z10950 [1 << 17]byte +var z10951 [1 << 17]byte +var z10952 [1 << 17]byte +var z10953 [1 << 17]byte +var z10954 [1 << 17]byte +var z10955 [1 << 17]byte +var z10956 [1 << 17]byte +var z10957 [1 << 17]byte +var z10958 [1 << 17]byte +var z10959 [1 << 17]byte +var z10960 [1 << 17]byte +var z10961 [1 << 17]byte +var z10962 [1 << 17]byte +var z10963 [1 << 17]byte +var z10964 [1 << 17]byte +var z10965 [1 << 17]byte +var z10966 [1 << 17]byte +var z10967 [1 << 17]byte +var z10968 [1 << 17]byte +var z10969 [1 << 17]byte +var z10970 [1 << 17]byte +var z10971 [1 << 17]byte +var z10972 [1 << 17]byte +var z10973 [1 << 17]byte +var z10974 [1 << 17]byte +var z10975 [1 << 17]byte +var z10976 [1 << 17]byte +var z10977 [1 << 17]byte +var z10978 [1 << 17]byte +var z10979 [1 << 17]byte +var z10980 [1 << 17]byte +var z10981 [1 << 17]byte +var z10982 [1 << 17]byte +var z10983 [1 << 17]byte +var z10984 [1 << 17]byte +var z10985 [1 << 17]byte +var z10986 [1 << 17]byte +var z10987 [1 << 17]byte +var z10988 [1 << 17]byte +var z10989 [1 << 17]byte +var z10990 [1 << 17]byte +var z10991 [1 << 17]byte +var z10992 [1 << 17]byte +var z10993 [1 << 17]byte +var z10994 [1 << 17]byte +var z10995 [1 << 17]byte +var z10996 [1 << 17]byte +var z10997 [1 << 17]byte +var z10998 [1 << 17]byte +var z10999 [1 << 17]byte +var z11000 [1 << 17]byte +var z11001 [1 << 17]byte +var z11002 [1 << 17]byte +var z11003 [1 << 17]byte +var z11004 [1 << 17]byte +var z11005 [1 << 17]byte +var z11006 [1 << 17]byte +var z11007 [1 << 17]byte +var z11008 [1 << 17]byte +var z11009 [1 << 17]byte +var z11010 [1 << 17]byte +var z11011 [1 << 17]byte +var z11012 [1 << 17]byte +var z11013 [1 << 17]byte +var z11014 [1 << 17]byte +var z11015 [1 << 17]byte +var z11016 [1 << 17]byte +var z11017 [1 << 17]byte +var z11018 [1 << 17]byte +var z11019 [1 << 17]byte +var z11020 [1 << 17]byte +var z11021 [1 << 17]byte +var z11022 [1 << 17]byte +var z11023 [1 << 17]byte +var z11024 [1 << 17]byte +var z11025 [1 << 17]byte +var z11026 [1 << 17]byte +var z11027 [1 << 17]byte +var z11028 [1 << 17]byte +var z11029 [1 << 17]byte +var z11030 [1 << 17]byte +var z11031 [1 << 17]byte +var z11032 [1 << 17]byte +var z11033 [1 << 17]byte +var z11034 [1 << 17]byte +var z11035 [1 << 17]byte +var z11036 [1 << 17]byte +var z11037 [1 << 17]byte +var z11038 [1 << 17]byte +var z11039 [1 << 17]byte +var z11040 [1 << 17]byte +var z11041 [1 << 17]byte +var z11042 [1 << 17]byte +var z11043 [1 << 17]byte +var z11044 [1 << 17]byte +var z11045 [1 << 17]byte +var z11046 [1 << 17]byte +var z11047 [1 << 17]byte +var z11048 [1 << 17]byte +var z11049 [1 << 17]byte +var z11050 [1 << 17]byte +var z11051 [1 << 17]byte +var z11052 [1 << 17]byte +var z11053 [1 << 17]byte +var z11054 [1 << 17]byte +var z11055 [1 << 17]byte +var z11056 [1 << 17]byte +var z11057 [1 << 17]byte +var z11058 [1 << 17]byte +var z11059 [1 << 17]byte +var z11060 [1 << 17]byte +var z11061 [1 << 17]byte +var z11062 [1 << 17]byte +var z11063 [1 << 17]byte +var z11064 [1 << 17]byte +var z11065 [1 << 17]byte +var z11066 [1 << 17]byte +var z11067 [1 << 17]byte +var z11068 [1 << 17]byte +var z11069 [1 << 17]byte +var z11070 [1 << 17]byte +var z11071 [1 << 17]byte +var z11072 [1 << 17]byte +var z11073 [1 << 17]byte +var z11074 [1 << 17]byte +var z11075 [1 << 17]byte +var z11076 [1 << 17]byte +var z11077 [1 << 17]byte +var z11078 [1 << 17]byte +var z11079 [1 << 17]byte +var z11080 [1 << 17]byte +var z11081 [1 << 17]byte +var z11082 [1 << 17]byte +var z11083 [1 << 17]byte +var z11084 [1 << 17]byte +var z11085 [1 << 17]byte +var z11086 [1 << 17]byte +var z11087 [1 << 17]byte +var z11088 [1 << 17]byte +var z11089 [1 << 17]byte +var z11090 [1 << 17]byte +var z11091 [1 << 17]byte +var z11092 [1 << 17]byte +var z11093 [1 << 17]byte +var z11094 [1 << 17]byte +var z11095 [1 << 17]byte +var z11096 [1 << 17]byte +var z11097 [1 << 17]byte +var z11098 [1 << 17]byte +var z11099 [1 << 17]byte +var z11100 [1 << 17]byte +var z11101 [1 << 17]byte +var z11102 [1 << 17]byte +var z11103 [1 << 17]byte +var z11104 [1 << 17]byte +var z11105 [1 << 17]byte +var z11106 [1 << 17]byte +var z11107 [1 << 17]byte +var z11108 [1 << 17]byte +var z11109 [1 << 17]byte +var z11110 [1 << 17]byte +var z11111 [1 << 17]byte +var z11112 [1 << 17]byte +var z11113 [1 << 17]byte +var z11114 [1 << 17]byte +var z11115 [1 << 17]byte +var z11116 [1 << 17]byte +var z11117 [1 << 17]byte +var z11118 [1 << 17]byte +var z11119 [1 << 17]byte +var z11120 [1 << 17]byte +var z11121 [1 << 17]byte +var z11122 [1 << 17]byte +var z11123 [1 << 17]byte +var z11124 [1 << 17]byte +var z11125 [1 << 17]byte +var z11126 [1 << 17]byte +var z11127 [1 << 17]byte +var z11128 [1 << 17]byte +var z11129 [1 << 17]byte +var z11130 [1 << 17]byte +var z11131 [1 << 17]byte +var z11132 [1 << 17]byte +var z11133 [1 << 17]byte +var z11134 [1 << 17]byte +var z11135 [1 << 17]byte +var z11136 [1 << 17]byte +var z11137 [1 << 17]byte +var z11138 [1 << 17]byte +var z11139 [1 << 17]byte +var z11140 [1 << 17]byte +var z11141 [1 << 17]byte +var z11142 [1 << 17]byte +var z11143 [1 << 17]byte +var z11144 [1 << 17]byte +var z11145 [1 << 17]byte +var z11146 [1 << 17]byte +var z11147 [1 << 17]byte +var z11148 [1 << 17]byte +var z11149 [1 << 17]byte +var z11150 [1 << 17]byte +var z11151 [1 << 17]byte +var z11152 [1 << 17]byte +var z11153 [1 << 17]byte +var z11154 [1 << 17]byte +var z11155 [1 << 17]byte +var z11156 [1 << 17]byte +var z11157 [1 << 17]byte +var z11158 [1 << 17]byte +var z11159 [1 << 17]byte +var z11160 [1 << 17]byte +var z11161 [1 << 17]byte +var z11162 [1 << 17]byte +var z11163 [1 << 17]byte +var z11164 [1 << 17]byte +var z11165 [1 << 17]byte +var z11166 [1 << 17]byte +var z11167 [1 << 17]byte +var z11168 [1 << 17]byte +var z11169 [1 << 17]byte +var z11170 [1 << 17]byte +var z11171 [1 << 17]byte +var z11172 [1 << 17]byte +var z11173 [1 << 17]byte +var z11174 [1 << 17]byte +var z11175 [1 << 17]byte +var z11176 [1 << 17]byte +var z11177 [1 << 17]byte +var z11178 [1 << 17]byte +var z11179 [1 << 17]byte +var z11180 [1 << 17]byte +var z11181 [1 << 17]byte +var z11182 [1 << 17]byte +var z11183 [1 << 17]byte +var z11184 [1 << 17]byte +var z11185 [1 << 17]byte +var z11186 [1 << 17]byte +var z11187 [1 << 17]byte +var z11188 [1 << 17]byte +var z11189 [1 << 17]byte +var z11190 [1 << 17]byte +var z11191 [1 << 17]byte +var z11192 [1 << 17]byte +var z11193 [1 << 17]byte +var z11194 [1 << 17]byte +var z11195 [1 << 17]byte +var z11196 [1 << 17]byte +var z11197 [1 << 17]byte +var z11198 [1 << 17]byte +var z11199 [1 << 17]byte +var z11200 [1 << 17]byte +var z11201 [1 << 17]byte +var z11202 [1 << 17]byte +var z11203 [1 << 17]byte +var z11204 [1 << 17]byte +var z11205 [1 << 17]byte +var z11206 [1 << 17]byte +var z11207 [1 << 17]byte +var z11208 [1 << 17]byte +var z11209 [1 << 17]byte +var z11210 [1 << 17]byte +var z11211 [1 << 17]byte +var z11212 [1 << 17]byte +var z11213 [1 << 17]byte +var z11214 [1 << 17]byte +var z11215 [1 << 17]byte +var z11216 [1 << 17]byte +var z11217 [1 << 17]byte +var z11218 [1 << 17]byte +var z11219 [1 << 17]byte +var z11220 [1 << 17]byte +var z11221 [1 << 17]byte +var z11222 [1 << 17]byte +var z11223 [1 << 17]byte +var z11224 [1 << 17]byte +var z11225 [1 << 17]byte +var z11226 [1 << 17]byte +var z11227 [1 << 17]byte +var z11228 [1 << 17]byte +var z11229 [1 << 17]byte +var z11230 [1 << 17]byte +var z11231 [1 << 17]byte +var z11232 [1 << 17]byte +var z11233 [1 << 17]byte +var z11234 [1 << 17]byte +var z11235 [1 << 17]byte +var z11236 [1 << 17]byte +var z11237 [1 << 17]byte +var z11238 [1 << 17]byte +var z11239 [1 << 17]byte +var z11240 [1 << 17]byte +var z11241 [1 << 17]byte +var z11242 [1 << 17]byte +var z11243 [1 << 17]byte +var z11244 [1 << 17]byte +var z11245 [1 << 17]byte +var z11246 [1 << 17]byte +var z11247 [1 << 17]byte +var z11248 [1 << 17]byte +var z11249 [1 << 17]byte +var z11250 [1 << 17]byte +var z11251 [1 << 17]byte +var z11252 [1 << 17]byte +var z11253 [1 << 17]byte +var z11254 [1 << 17]byte +var z11255 [1 << 17]byte +var z11256 [1 << 17]byte +var z11257 [1 << 17]byte +var z11258 [1 << 17]byte +var z11259 [1 << 17]byte +var z11260 [1 << 17]byte +var z11261 [1 << 17]byte +var z11262 [1 << 17]byte +var z11263 [1 << 17]byte +var z11264 [1 << 17]byte +var z11265 [1 << 17]byte +var z11266 [1 << 17]byte +var z11267 [1 << 17]byte +var z11268 [1 << 17]byte +var z11269 [1 << 17]byte +var z11270 [1 << 17]byte +var z11271 [1 << 17]byte +var z11272 [1 << 17]byte +var z11273 [1 << 17]byte +var z11274 [1 << 17]byte +var z11275 [1 << 17]byte +var z11276 [1 << 17]byte +var z11277 [1 << 17]byte +var z11278 [1 << 17]byte +var z11279 [1 << 17]byte +var z11280 [1 << 17]byte +var z11281 [1 << 17]byte +var z11282 [1 << 17]byte +var z11283 [1 << 17]byte +var z11284 [1 << 17]byte +var z11285 [1 << 17]byte +var z11286 [1 << 17]byte +var z11287 [1 << 17]byte +var z11288 [1 << 17]byte +var z11289 [1 << 17]byte +var z11290 [1 << 17]byte +var z11291 [1 << 17]byte +var z11292 [1 << 17]byte +var z11293 [1 << 17]byte +var z11294 [1 << 17]byte +var z11295 [1 << 17]byte +var z11296 [1 << 17]byte +var z11297 [1 << 17]byte +var z11298 [1 << 17]byte +var z11299 [1 << 17]byte +var z11300 [1 << 17]byte +var z11301 [1 << 17]byte +var z11302 [1 << 17]byte +var z11303 [1 << 17]byte +var z11304 [1 << 17]byte +var z11305 [1 << 17]byte +var z11306 [1 << 17]byte +var z11307 [1 << 17]byte +var z11308 [1 << 17]byte +var z11309 [1 << 17]byte +var z11310 [1 << 17]byte +var z11311 [1 << 17]byte +var z11312 [1 << 17]byte +var z11313 [1 << 17]byte +var z11314 [1 << 17]byte +var z11315 [1 << 17]byte +var z11316 [1 << 17]byte +var z11317 [1 << 17]byte +var z11318 [1 << 17]byte +var z11319 [1 << 17]byte +var z11320 [1 << 17]byte +var z11321 [1 << 17]byte +var z11322 [1 << 17]byte +var z11323 [1 << 17]byte +var z11324 [1 << 17]byte +var z11325 [1 << 17]byte +var z11326 [1 << 17]byte +var z11327 [1 << 17]byte +var z11328 [1 << 17]byte +var z11329 [1 << 17]byte +var z11330 [1 << 17]byte +var z11331 [1 << 17]byte +var z11332 [1 << 17]byte +var z11333 [1 << 17]byte +var z11334 [1 << 17]byte +var z11335 [1 << 17]byte +var z11336 [1 << 17]byte +var z11337 [1 << 17]byte +var z11338 [1 << 17]byte +var z11339 [1 << 17]byte +var z11340 [1 << 17]byte +var z11341 [1 << 17]byte +var z11342 [1 << 17]byte +var z11343 [1 << 17]byte +var z11344 [1 << 17]byte +var z11345 [1 << 17]byte +var z11346 [1 << 17]byte +var z11347 [1 << 17]byte +var z11348 [1 << 17]byte +var z11349 [1 << 17]byte +var z11350 [1 << 17]byte +var z11351 [1 << 17]byte +var z11352 [1 << 17]byte +var z11353 [1 << 17]byte +var z11354 [1 << 17]byte +var z11355 [1 << 17]byte +var z11356 [1 << 17]byte +var z11357 [1 << 17]byte +var z11358 [1 << 17]byte +var z11359 [1 << 17]byte +var z11360 [1 << 17]byte +var z11361 [1 << 17]byte +var z11362 [1 << 17]byte +var z11363 [1 << 17]byte +var z11364 [1 << 17]byte +var z11365 [1 << 17]byte +var z11366 [1 << 17]byte +var z11367 [1 << 17]byte +var z11368 [1 << 17]byte +var z11369 [1 << 17]byte +var z11370 [1 << 17]byte +var z11371 [1 << 17]byte +var z11372 [1 << 17]byte +var z11373 [1 << 17]byte +var z11374 [1 << 17]byte +var z11375 [1 << 17]byte +var z11376 [1 << 17]byte +var z11377 [1 << 17]byte +var z11378 [1 << 17]byte +var z11379 [1 << 17]byte +var z11380 [1 << 17]byte +var z11381 [1 << 17]byte +var z11382 [1 << 17]byte +var z11383 [1 << 17]byte +var z11384 [1 << 17]byte +var z11385 [1 << 17]byte +var z11386 [1 << 17]byte +var z11387 [1 << 17]byte +var z11388 [1 << 17]byte +var z11389 [1 << 17]byte +var z11390 [1 << 17]byte +var z11391 [1 << 17]byte +var z11392 [1 << 17]byte +var z11393 [1 << 17]byte +var z11394 [1 << 17]byte +var z11395 [1 << 17]byte +var z11396 [1 << 17]byte +var z11397 [1 << 17]byte +var z11398 [1 << 17]byte +var z11399 [1 << 17]byte +var z11400 [1 << 17]byte +var z11401 [1 << 17]byte +var z11402 [1 << 17]byte +var z11403 [1 << 17]byte +var z11404 [1 << 17]byte +var z11405 [1 << 17]byte +var z11406 [1 << 17]byte +var z11407 [1 << 17]byte +var z11408 [1 << 17]byte +var z11409 [1 << 17]byte +var z11410 [1 << 17]byte +var z11411 [1 << 17]byte +var z11412 [1 << 17]byte +var z11413 [1 << 17]byte +var z11414 [1 << 17]byte +var z11415 [1 << 17]byte +var z11416 [1 << 17]byte +var z11417 [1 << 17]byte +var z11418 [1 << 17]byte +var z11419 [1 << 17]byte +var z11420 [1 << 17]byte +var z11421 [1 << 17]byte +var z11422 [1 << 17]byte +var z11423 [1 << 17]byte +var z11424 [1 << 17]byte +var z11425 [1 << 17]byte +var z11426 [1 << 17]byte +var z11427 [1 << 17]byte +var z11428 [1 << 17]byte +var z11429 [1 << 17]byte +var z11430 [1 << 17]byte +var z11431 [1 << 17]byte +var z11432 [1 << 17]byte +var z11433 [1 << 17]byte +var z11434 [1 << 17]byte +var z11435 [1 << 17]byte +var z11436 [1 << 17]byte +var z11437 [1 << 17]byte +var z11438 [1 << 17]byte +var z11439 [1 << 17]byte +var z11440 [1 << 17]byte +var z11441 [1 << 17]byte +var z11442 [1 << 17]byte +var z11443 [1 << 17]byte +var z11444 [1 << 17]byte +var z11445 [1 << 17]byte +var z11446 [1 << 17]byte +var z11447 [1 << 17]byte +var z11448 [1 << 17]byte +var z11449 [1 << 17]byte +var z11450 [1 << 17]byte +var z11451 [1 << 17]byte +var z11452 [1 << 17]byte +var z11453 [1 << 17]byte +var z11454 [1 << 17]byte +var z11455 [1 << 17]byte +var z11456 [1 << 17]byte +var z11457 [1 << 17]byte +var z11458 [1 << 17]byte +var z11459 [1 << 17]byte +var z11460 [1 << 17]byte +var z11461 [1 << 17]byte +var z11462 [1 << 17]byte +var z11463 [1 << 17]byte +var z11464 [1 << 17]byte +var z11465 [1 << 17]byte +var z11466 [1 << 17]byte +var z11467 [1 << 17]byte +var z11468 [1 << 17]byte +var z11469 [1 << 17]byte +var z11470 [1 << 17]byte +var z11471 [1 << 17]byte +var z11472 [1 << 17]byte +var z11473 [1 << 17]byte +var z11474 [1 << 17]byte +var z11475 [1 << 17]byte +var z11476 [1 << 17]byte +var z11477 [1 << 17]byte +var z11478 [1 << 17]byte +var z11479 [1 << 17]byte +var z11480 [1 << 17]byte +var z11481 [1 << 17]byte +var z11482 [1 << 17]byte +var z11483 [1 << 17]byte +var z11484 [1 << 17]byte +var z11485 [1 << 17]byte +var z11486 [1 << 17]byte +var z11487 [1 << 17]byte +var z11488 [1 << 17]byte +var z11489 [1 << 17]byte +var z11490 [1 << 17]byte +var z11491 [1 << 17]byte +var z11492 [1 << 17]byte +var z11493 [1 << 17]byte +var z11494 [1 << 17]byte +var z11495 [1 << 17]byte +var z11496 [1 << 17]byte +var z11497 [1 << 17]byte +var z11498 [1 << 17]byte +var z11499 [1 << 17]byte +var z11500 [1 << 17]byte +var z11501 [1 << 17]byte +var z11502 [1 << 17]byte +var z11503 [1 << 17]byte +var z11504 [1 << 17]byte +var z11505 [1 << 17]byte +var z11506 [1 << 17]byte +var z11507 [1 << 17]byte +var z11508 [1 << 17]byte +var z11509 [1 << 17]byte +var z11510 [1 << 17]byte +var z11511 [1 << 17]byte +var z11512 [1 << 17]byte +var z11513 [1 << 17]byte +var z11514 [1 << 17]byte +var z11515 [1 << 17]byte +var z11516 [1 << 17]byte +var z11517 [1 << 17]byte +var z11518 [1 << 17]byte +var z11519 [1 << 17]byte +var z11520 [1 << 17]byte +var z11521 [1 << 17]byte +var z11522 [1 << 17]byte +var z11523 [1 << 17]byte +var z11524 [1 << 17]byte +var z11525 [1 << 17]byte +var z11526 [1 << 17]byte +var z11527 [1 << 17]byte +var z11528 [1 << 17]byte +var z11529 [1 << 17]byte +var z11530 [1 << 17]byte +var z11531 [1 << 17]byte +var z11532 [1 << 17]byte +var z11533 [1 << 17]byte +var z11534 [1 << 17]byte +var z11535 [1 << 17]byte +var z11536 [1 << 17]byte +var z11537 [1 << 17]byte +var z11538 [1 << 17]byte +var z11539 [1 << 17]byte +var z11540 [1 << 17]byte +var z11541 [1 << 17]byte +var z11542 [1 << 17]byte +var z11543 [1 << 17]byte +var z11544 [1 << 17]byte +var z11545 [1 << 17]byte +var z11546 [1 << 17]byte +var z11547 [1 << 17]byte +var z11548 [1 << 17]byte +var z11549 [1 << 17]byte +var z11550 [1 << 17]byte +var z11551 [1 << 17]byte +var z11552 [1 << 17]byte +var z11553 [1 << 17]byte +var z11554 [1 << 17]byte +var z11555 [1 << 17]byte +var z11556 [1 << 17]byte +var z11557 [1 << 17]byte +var z11558 [1 << 17]byte +var z11559 [1 << 17]byte +var z11560 [1 << 17]byte +var z11561 [1 << 17]byte +var z11562 [1 << 17]byte +var z11563 [1 << 17]byte +var z11564 [1 << 17]byte +var z11565 [1 << 17]byte +var z11566 [1 << 17]byte +var z11567 [1 << 17]byte +var z11568 [1 << 17]byte +var z11569 [1 << 17]byte +var z11570 [1 << 17]byte +var z11571 [1 << 17]byte +var z11572 [1 << 17]byte +var z11573 [1 << 17]byte +var z11574 [1 << 17]byte +var z11575 [1 << 17]byte +var z11576 [1 << 17]byte +var z11577 [1 << 17]byte +var z11578 [1 << 17]byte +var z11579 [1 << 17]byte +var z11580 [1 << 17]byte +var z11581 [1 << 17]byte +var z11582 [1 << 17]byte +var z11583 [1 << 17]byte +var z11584 [1 << 17]byte +var z11585 [1 << 17]byte +var z11586 [1 << 17]byte +var z11587 [1 << 17]byte +var z11588 [1 << 17]byte +var z11589 [1 << 17]byte +var z11590 [1 << 17]byte +var z11591 [1 << 17]byte +var z11592 [1 << 17]byte +var z11593 [1 << 17]byte +var z11594 [1 << 17]byte +var z11595 [1 << 17]byte +var z11596 [1 << 17]byte +var z11597 [1 << 17]byte +var z11598 [1 << 17]byte +var z11599 [1 << 17]byte +var z11600 [1 << 17]byte +var z11601 [1 << 17]byte +var z11602 [1 << 17]byte +var z11603 [1 << 17]byte +var z11604 [1 << 17]byte +var z11605 [1 << 17]byte +var z11606 [1 << 17]byte +var z11607 [1 << 17]byte +var z11608 [1 << 17]byte +var z11609 [1 << 17]byte +var z11610 [1 << 17]byte +var z11611 [1 << 17]byte +var z11612 [1 << 17]byte +var z11613 [1 << 17]byte +var z11614 [1 << 17]byte +var z11615 [1 << 17]byte +var z11616 [1 << 17]byte +var z11617 [1 << 17]byte +var z11618 [1 << 17]byte +var z11619 [1 << 17]byte +var z11620 [1 << 17]byte +var z11621 [1 << 17]byte +var z11622 [1 << 17]byte +var z11623 [1 << 17]byte +var z11624 [1 << 17]byte +var z11625 [1 << 17]byte +var z11626 [1 << 17]byte +var z11627 [1 << 17]byte +var z11628 [1 << 17]byte +var z11629 [1 << 17]byte +var z11630 [1 << 17]byte +var z11631 [1 << 17]byte +var z11632 [1 << 17]byte +var z11633 [1 << 17]byte +var z11634 [1 << 17]byte +var z11635 [1 << 17]byte +var z11636 [1 << 17]byte +var z11637 [1 << 17]byte +var z11638 [1 << 17]byte +var z11639 [1 << 17]byte +var z11640 [1 << 17]byte +var z11641 [1 << 17]byte +var z11642 [1 << 17]byte +var z11643 [1 << 17]byte +var z11644 [1 << 17]byte +var z11645 [1 << 17]byte +var z11646 [1 << 17]byte +var z11647 [1 << 17]byte +var z11648 [1 << 17]byte +var z11649 [1 << 17]byte +var z11650 [1 << 17]byte +var z11651 [1 << 17]byte +var z11652 [1 << 17]byte +var z11653 [1 << 17]byte +var z11654 [1 << 17]byte +var z11655 [1 << 17]byte +var z11656 [1 << 17]byte +var z11657 [1 << 17]byte +var z11658 [1 << 17]byte +var z11659 [1 << 17]byte +var z11660 [1 << 17]byte +var z11661 [1 << 17]byte +var z11662 [1 << 17]byte +var z11663 [1 << 17]byte +var z11664 [1 << 17]byte +var z11665 [1 << 17]byte +var z11666 [1 << 17]byte +var z11667 [1 << 17]byte +var z11668 [1 << 17]byte +var z11669 [1 << 17]byte +var z11670 [1 << 17]byte +var z11671 [1 << 17]byte +var z11672 [1 << 17]byte +var z11673 [1 << 17]byte +var z11674 [1 << 17]byte +var z11675 [1 << 17]byte +var z11676 [1 << 17]byte +var z11677 [1 << 17]byte +var z11678 [1 << 17]byte +var z11679 [1 << 17]byte +var z11680 [1 << 17]byte +var z11681 [1 << 17]byte +var z11682 [1 << 17]byte +var z11683 [1 << 17]byte +var z11684 [1 << 17]byte +var z11685 [1 << 17]byte +var z11686 [1 << 17]byte +var z11687 [1 << 17]byte +var z11688 [1 << 17]byte +var z11689 [1 << 17]byte +var z11690 [1 << 17]byte +var z11691 [1 << 17]byte +var z11692 [1 << 17]byte +var z11693 [1 << 17]byte +var z11694 [1 << 17]byte +var z11695 [1 << 17]byte +var z11696 [1 << 17]byte +var z11697 [1 << 17]byte +var z11698 [1 << 17]byte +var z11699 [1 << 17]byte +var z11700 [1 << 17]byte +var z11701 [1 << 17]byte +var z11702 [1 << 17]byte +var z11703 [1 << 17]byte +var z11704 [1 << 17]byte +var z11705 [1 << 17]byte +var z11706 [1 << 17]byte +var z11707 [1 << 17]byte +var z11708 [1 << 17]byte +var z11709 [1 << 17]byte +var z11710 [1 << 17]byte +var z11711 [1 << 17]byte +var z11712 [1 << 17]byte +var z11713 [1 << 17]byte +var z11714 [1 << 17]byte +var z11715 [1 << 17]byte +var z11716 [1 << 17]byte +var z11717 [1 << 17]byte +var z11718 [1 << 17]byte +var z11719 [1 << 17]byte +var z11720 [1 << 17]byte +var z11721 [1 << 17]byte +var z11722 [1 << 17]byte +var z11723 [1 << 17]byte +var z11724 [1 << 17]byte +var z11725 [1 << 17]byte +var z11726 [1 << 17]byte +var z11727 [1 << 17]byte +var z11728 [1 << 17]byte +var z11729 [1 << 17]byte +var z11730 [1 << 17]byte +var z11731 [1 << 17]byte +var z11732 [1 << 17]byte +var z11733 [1 << 17]byte +var z11734 [1 << 17]byte +var z11735 [1 << 17]byte +var z11736 [1 << 17]byte +var z11737 [1 << 17]byte +var z11738 [1 << 17]byte +var z11739 [1 << 17]byte +var z11740 [1 << 17]byte +var z11741 [1 << 17]byte +var z11742 [1 << 17]byte +var z11743 [1 << 17]byte +var z11744 [1 << 17]byte +var z11745 [1 << 17]byte +var z11746 [1 << 17]byte +var z11747 [1 << 17]byte +var z11748 [1 << 17]byte +var z11749 [1 << 17]byte +var z11750 [1 << 17]byte +var z11751 [1 << 17]byte +var z11752 [1 << 17]byte +var z11753 [1 << 17]byte +var z11754 [1 << 17]byte +var z11755 [1 << 17]byte +var z11756 [1 << 17]byte +var z11757 [1 << 17]byte +var z11758 [1 << 17]byte +var z11759 [1 << 17]byte +var z11760 [1 << 17]byte +var z11761 [1 << 17]byte +var z11762 [1 << 17]byte +var z11763 [1 << 17]byte +var z11764 [1 << 17]byte +var z11765 [1 << 17]byte +var z11766 [1 << 17]byte +var z11767 [1 << 17]byte +var z11768 [1 << 17]byte +var z11769 [1 << 17]byte +var z11770 [1 << 17]byte +var z11771 [1 << 17]byte +var z11772 [1 << 17]byte +var z11773 [1 << 17]byte +var z11774 [1 << 17]byte +var z11775 [1 << 17]byte +var z11776 [1 << 17]byte +var z11777 [1 << 17]byte +var z11778 [1 << 17]byte +var z11779 [1 << 17]byte +var z11780 [1 << 17]byte +var z11781 [1 << 17]byte +var z11782 [1 << 17]byte +var z11783 [1 << 17]byte +var z11784 [1 << 17]byte +var z11785 [1 << 17]byte +var z11786 [1 << 17]byte +var z11787 [1 << 17]byte +var z11788 [1 << 17]byte +var z11789 [1 << 17]byte +var z11790 [1 << 17]byte +var z11791 [1 << 17]byte +var z11792 [1 << 17]byte +var z11793 [1 << 17]byte +var z11794 [1 << 17]byte +var z11795 [1 << 17]byte +var z11796 [1 << 17]byte +var z11797 [1 << 17]byte +var z11798 [1 << 17]byte +var z11799 [1 << 17]byte +var z11800 [1 << 17]byte +var z11801 [1 << 17]byte +var z11802 [1 << 17]byte +var z11803 [1 << 17]byte +var z11804 [1 << 17]byte +var z11805 [1 << 17]byte +var z11806 [1 << 17]byte +var z11807 [1 << 17]byte +var z11808 [1 << 17]byte +var z11809 [1 << 17]byte +var z11810 [1 << 17]byte +var z11811 [1 << 17]byte +var z11812 [1 << 17]byte +var z11813 [1 << 17]byte +var z11814 [1 << 17]byte +var z11815 [1 << 17]byte +var z11816 [1 << 17]byte +var z11817 [1 << 17]byte +var z11818 [1 << 17]byte +var z11819 [1 << 17]byte +var z11820 [1 << 17]byte +var z11821 [1 << 17]byte +var z11822 [1 << 17]byte +var z11823 [1 << 17]byte +var z11824 [1 << 17]byte +var z11825 [1 << 17]byte +var z11826 [1 << 17]byte +var z11827 [1 << 17]byte +var z11828 [1 << 17]byte +var z11829 [1 << 17]byte +var z11830 [1 << 17]byte +var z11831 [1 << 17]byte +var z11832 [1 << 17]byte +var z11833 [1 << 17]byte +var z11834 [1 << 17]byte +var z11835 [1 << 17]byte +var z11836 [1 << 17]byte +var z11837 [1 << 17]byte +var z11838 [1 << 17]byte +var z11839 [1 << 17]byte +var z11840 [1 << 17]byte +var z11841 [1 << 17]byte +var z11842 [1 << 17]byte +var z11843 [1 << 17]byte +var z11844 [1 << 17]byte +var z11845 [1 << 17]byte +var z11846 [1 << 17]byte +var z11847 [1 << 17]byte +var z11848 [1 << 17]byte +var z11849 [1 << 17]byte +var z11850 [1 << 17]byte +var z11851 [1 << 17]byte +var z11852 [1 << 17]byte +var z11853 [1 << 17]byte +var z11854 [1 << 17]byte +var z11855 [1 << 17]byte +var z11856 [1 << 17]byte +var z11857 [1 << 17]byte +var z11858 [1 << 17]byte +var z11859 [1 << 17]byte +var z11860 [1 << 17]byte +var z11861 [1 << 17]byte +var z11862 [1 << 17]byte +var z11863 [1 << 17]byte +var z11864 [1 << 17]byte +var z11865 [1 << 17]byte +var z11866 [1 << 17]byte +var z11867 [1 << 17]byte +var z11868 [1 << 17]byte +var z11869 [1 << 17]byte +var z11870 [1 << 17]byte +var z11871 [1 << 17]byte +var z11872 [1 << 17]byte +var z11873 [1 << 17]byte +var z11874 [1 << 17]byte +var z11875 [1 << 17]byte +var z11876 [1 << 17]byte +var z11877 [1 << 17]byte +var z11878 [1 << 17]byte +var z11879 [1 << 17]byte +var z11880 [1 << 17]byte +var z11881 [1 << 17]byte +var z11882 [1 << 17]byte +var z11883 [1 << 17]byte +var z11884 [1 << 17]byte +var z11885 [1 << 17]byte +var z11886 [1 << 17]byte +var z11887 [1 << 17]byte +var z11888 [1 << 17]byte +var z11889 [1 << 17]byte +var z11890 [1 << 17]byte +var z11891 [1 << 17]byte +var z11892 [1 << 17]byte +var z11893 [1 << 17]byte +var z11894 [1 << 17]byte +var z11895 [1 << 17]byte +var z11896 [1 << 17]byte +var z11897 [1 << 17]byte +var z11898 [1 << 17]byte +var z11899 [1 << 17]byte +var z11900 [1 << 17]byte +var z11901 [1 << 17]byte +var z11902 [1 << 17]byte +var z11903 [1 << 17]byte +var z11904 [1 << 17]byte +var z11905 [1 << 17]byte +var z11906 [1 << 17]byte +var z11907 [1 << 17]byte +var z11908 [1 << 17]byte +var z11909 [1 << 17]byte +var z11910 [1 << 17]byte +var z11911 [1 << 17]byte +var z11912 [1 << 17]byte +var z11913 [1 << 17]byte +var z11914 [1 << 17]byte +var z11915 [1 << 17]byte +var z11916 [1 << 17]byte +var z11917 [1 << 17]byte +var z11918 [1 << 17]byte +var z11919 [1 << 17]byte +var z11920 [1 << 17]byte +var z11921 [1 << 17]byte +var z11922 [1 << 17]byte +var z11923 [1 << 17]byte +var z11924 [1 << 17]byte +var z11925 [1 << 17]byte +var z11926 [1 << 17]byte +var z11927 [1 << 17]byte +var z11928 [1 << 17]byte +var z11929 [1 << 17]byte +var z11930 [1 << 17]byte +var z11931 [1 << 17]byte +var z11932 [1 << 17]byte +var z11933 [1 << 17]byte +var z11934 [1 << 17]byte +var z11935 [1 << 17]byte +var z11936 [1 << 17]byte +var z11937 [1 << 17]byte +var z11938 [1 << 17]byte +var z11939 [1 << 17]byte +var z11940 [1 << 17]byte +var z11941 [1 << 17]byte +var z11942 [1 << 17]byte +var z11943 [1 << 17]byte +var z11944 [1 << 17]byte +var z11945 [1 << 17]byte +var z11946 [1 << 17]byte +var z11947 [1 << 17]byte +var z11948 [1 << 17]byte +var z11949 [1 << 17]byte +var z11950 [1 << 17]byte +var z11951 [1 << 17]byte +var z11952 [1 << 17]byte +var z11953 [1 << 17]byte +var z11954 [1 << 17]byte +var z11955 [1 << 17]byte +var z11956 [1 << 17]byte +var z11957 [1 << 17]byte +var z11958 [1 << 17]byte +var z11959 [1 << 17]byte +var z11960 [1 << 17]byte +var z11961 [1 << 17]byte +var z11962 [1 << 17]byte +var z11963 [1 << 17]byte +var z11964 [1 << 17]byte +var z11965 [1 << 17]byte +var z11966 [1 << 17]byte +var z11967 [1 << 17]byte +var z11968 [1 << 17]byte +var z11969 [1 << 17]byte +var z11970 [1 << 17]byte +var z11971 [1 << 17]byte +var z11972 [1 << 17]byte +var z11973 [1 << 17]byte +var z11974 [1 << 17]byte +var z11975 [1 << 17]byte +var z11976 [1 << 17]byte +var z11977 [1 << 17]byte +var z11978 [1 << 17]byte +var z11979 [1 << 17]byte +var z11980 [1 << 17]byte +var z11981 [1 << 17]byte +var z11982 [1 << 17]byte +var z11983 [1 << 17]byte +var z11984 [1 << 17]byte +var z11985 [1 << 17]byte +var z11986 [1 << 17]byte +var z11987 [1 << 17]byte +var z11988 [1 << 17]byte +var z11989 [1 << 17]byte +var z11990 [1 << 17]byte +var z11991 [1 << 17]byte +var z11992 [1 << 17]byte +var z11993 [1 << 17]byte +var z11994 [1 << 17]byte +var z11995 [1 << 17]byte +var z11996 [1 << 17]byte +var z11997 [1 << 17]byte +var z11998 [1 << 17]byte +var z11999 [1 << 17]byte +var z12000 [1 << 17]byte +var z12001 [1 << 17]byte +var z12002 [1 << 17]byte +var z12003 [1 << 17]byte +var z12004 [1 << 17]byte +var z12005 [1 << 17]byte +var z12006 [1 << 17]byte +var z12007 [1 << 17]byte +var z12008 [1 << 17]byte +var z12009 [1 << 17]byte +var z12010 [1 << 17]byte +var z12011 [1 << 17]byte +var z12012 [1 << 17]byte +var z12013 [1 << 17]byte +var z12014 [1 << 17]byte +var z12015 [1 << 17]byte +var z12016 [1 << 17]byte +var z12017 [1 << 17]byte +var z12018 [1 << 17]byte +var z12019 [1 << 17]byte +var z12020 [1 << 17]byte +var z12021 [1 << 17]byte +var z12022 [1 << 17]byte +var z12023 [1 << 17]byte +var z12024 [1 << 17]byte +var z12025 [1 << 17]byte +var z12026 [1 << 17]byte +var z12027 [1 << 17]byte +var z12028 [1 << 17]byte +var z12029 [1 << 17]byte +var z12030 [1 << 17]byte +var z12031 [1 << 17]byte +var z12032 [1 << 17]byte +var z12033 [1 << 17]byte +var z12034 [1 << 17]byte +var z12035 [1 << 17]byte +var z12036 [1 << 17]byte +var z12037 [1 << 17]byte +var z12038 [1 << 17]byte +var z12039 [1 << 17]byte +var z12040 [1 << 17]byte +var z12041 [1 << 17]byte +var z12042 [1 << 17]byte +var z12043 [1 << 17]byte +var z12044 [1 << 17]byte +var z12045 [1 << 17]byte +var z12046 [1 << 17]byte +var z12047 [1 << 17]byte +var z12048 [1 << 17]byte +var z12049 [1 << 17]byte +var z12050 [1 << 17]byte +var z12051 [1 << 17]byte +var z12052 [1 << 17]byte +var z12053 [1 << 17]byte +var z12054 [1 << 17]byte +var z12055 [1 << 17]byte +var z12056 [1 << 17]byte +var z12057 [1 << 17]byte +var z12058 [1 << 17]byte +var z12059 [1 << 17]byte +var z12060 [1 << 17]byte +var z12061 [1 << 17]byte +var z12062 [1 << 17]byte +var z12063 [1 << 17]byte +var z12064 [1 << 17]byte +var z12065 [1 << 17]byte +var z12066 [1 << 17]byte +var z12067 [1 << 17]byte +var z12068 [1 << 17]byte +var z12069 [1 << 17]byte +var z12070 [1 << 17]byte +var z12071 [1 << 17]byte +var z12072 [1 << 17]byte +var z12073 [1 << 17]byte +var z12074 [1 << 17]byte +var z12075 [1 << 17]byte +var z12076 [1 << 17]byte +var z12077 [1 << 17]byte +var z12078 [1 << 17]byte +var z12079 [1 << 17]byte +var z12080 [1 << 17]byte +var z12081 [1 << 17]byte +var z12082 [1 << 17]byte +var z12083 [1 << 17]byte +var z12084 [1 << 17]byte +var z12085 [1 << 17]byte +var z12086 [1 << 17]byte +var z12087 [1 << 17]byte +var z12088 [1 << 17]byte +var z12089 [1 << 17]byte +var z12090 [1 << 17]byte +var z12091 [1 << 17]byte +var z12092 [1 << 17]byte +var z12093 [1 << 17]byte +var z12094 [1 << 17]byte +var z12095 [1 << 17]byte +var z12096 [1 << 17]byte +var z12097 [1 << 17]byte +var z12098 [1 << 17]byte +var z12099 [1 << 17]byte +var z12100 [1 << 17]byte +var z12101 [1 << 17]byte +var z12102 [1 << 17]byte +var z12103 [1 << 17]byte +var z12104 [1 << 17]byte +var z12105 [1 << 17]byte +var z12106 [1 << 17]byte +var z12107 [1 << 17]byte +var z12108 [1 << 17]byte +var z12109 [1 << 17]byte +var z12110 [1 << 17]byte +var z12111 [1 << 17]byte +var z12112 [1 << 17]byte +var z12113 [1 << 17]byte +var z12114 [1 << 17]byte +var z12115 [1 << 17]byte +var z12116 [1 << 17]byte +var z12117 [1 << 17]byte +var z12118 [1 << 17]byte +var z12119 [1 << 17]byte +var z12120 [1 << 17]byte +var z12121 [1 << 17]byte +var z12122 [1 << 17]byte +var z12123 [1 << 17]byte +var z12124 [1 << 17]byte +var z12125 [1 << 17]byte +var z12126 [1 << 17]byte +var z12127 [1 << 17]byte +var z12128 [1 << 17]byte +var z12129 [1 << 17]byte +var z12130 [1 << 17]byte +var z12131 [1 << 17]byte +var z12132 [1 << 17]byte +var z12133 [1 << 17]byte +var z12134 [1 << 17]byte +var z12135 [1 << 17]byte +var z12136 [1 << 17]byte +var z12137 [1 << 17]byte +var z12138 [1 << 17]byte +var z12139 [1 << 17]byte +var z12140 [1 << 17]byte +var z12141 [1 << 17]byte +var z12142 [1 << 17]byte +var z12143 [1 << 17]byte +var z12144 [1 << 17]byte +var z12145 [1 << 17]byte +var z12146 [1 << 17]byte +var z12147 [1 << 17]byte +var z12148 [1 << 17]byte +var z12149 [1 << 17]byte +var z12150 [1 << 17]byte +var z12151 [1 << 17]byte +var z12152 [1 << 17]byte +var z12153 [1 << 17]byte +var z12154 [1 << 17]byte +var z12155 [1 << 17]byte +var z12156 [1 << 17]byte +var z12157 [1 << 17]byte +var z12158 [1 << 17]byte +var z12159 [1 << 17]byte +var z12160 [1 << 17]byte +var z12161 [1 << 17]byte +var z12162 [1 << 17]byte +var z12163 [1 << 17]byte +var z12164 [1 << 17]byte +var z12165 [1 << 17]byte +var z12166 [1 << 17]byte +var z12167 [1 << 17]byte +var z12168 [1 << 17]byte +var z12169 [1 << 17]byte +var z12170 [1 << 17]byte +var z12171 [1 << 17]byte +var z12172 [1 << 17]byte +var z12173 [1 << 17]byte +var z12174 [1 << 17]byte +var z12175 [1 << 17]byte +var z12176 [1 << 17]byte +var z12177 [1 << 17]byte +var z12178 [1 << 17]byte +var z12179 [1 << 17]byte +var z12180 [1 << 17]byte +var z12181 [1 << 17]byte +var z12182 [1 << 17]byte +var z12183 [1 << 17]byte +var z12184 [1 << 17]byte +var z12185 [1 << 17]byte +var z12186 [1 << 17]byte +var z12187 [1 << 17]byte +var z12188 [1 << 17]byte +var z12189 [1 << 17]byte +var z12190 [1 << 17]byte +var z12191 [1 << 17]byte +var z12192 [1 << 17]byte +var z12193 [1 << 17]byte +var z12194 [1 << 17]byte +var z12195 [1 << 17]byte +var z12196 [1 << 17]byte +var z12197 [1 << 17]byte +var z12198 [1 << 17]byte +var z12199 [1 << 17]byte +var z12200 [1 << 17]byte +var z12201 [1 << 17]byte +var z12202 [1 << 17]byte +var z12203 [1 << 17]byte +var z12204 [1 << 17]byte +var z12205 [1 << 17]byte +var z12206 [1 << 17]byte +var z12207 [1 << 17]byte +var z12208 [1 << 17]byte +var z12209 [1 << 17]byte +var z12210 [1 << 17]byte +var z12211 [1 << 17]byte +var z12212 [1 << 17]byte +var z12213 [1 << 17]byte +var z12214 [1 << 17]byte +var z12215 [1 << 17]byte +var z12216 [1 << 17]byte +var z12217 [1 << 17]byte +var z12218 [1 << 17]byte +var z12219 [1 << 17]byte +var z12220 [1 << 17]byte +var z12221 [1 << 17]byte +var z12222 [1 << 17]byte +var z12223 [1 << 17]byte +var z12224 [1 << 17]byte +var z12225 [1 << 17]byte +var z12226 [1 << 17]byte +var z12227 [1 << 17]byte +var z12228 [1 << 17]byte +var z12229 [1 << 17]byte +var z12230 [1 << 17]byte +var z12231 [1 << 17]byte +var z12232 [1 << 17]byte +var z12233 [1 << 17]byte +var z12234 [1 << 17]byte +var z12235 [1 << 17]byte +var z12236 [1 << 17]byte +var z12237 [1 << 17]byte +var z12238 [1 << 17]byte +var z12239 [1 << 17]byte +var z12240 [1 << 17]byte +var z12241 [1 << 17]byte +var z12242 [1 << 17]byte +var z12243 [1 << 17]byte +var z12244 [1 << 17]byte +var z12245 [1 << 17]byte +var z12246 [1 << 17]byte +var z12247 [1 << 17]byte +var z12248 [1 << 17]byte +var z12249 [1 << 17]byte +var z12250 [1 << 17]byte +var z12251 [1 << 17]byte +var z12252 [1 << 17]byte +var z12253 [1 << 17]byte +var z12254 [1 << 17]byte +var z12255 [1 << 17]byte +var z12256 [1 << 17]byte +var z12257 [1 << 17]byte +var z12258 [1 << 17]byte +var z12259 [1 << 17]byte +var z12260 [1 << 17]byte +var z12261 [1 << 17]byte +var z12262 [1 << 17]byte +var z12263 [1 << 17]byte +var z12264 [1 << 17]byte +var z12265 [1 << 17]byte +var z12266 [1 << 17]byte +var z12267 [1 << 17]byte +var z12268 [1 << 17]byte +var z12269 [1 << 17]byte +var z12270 [1 << 17]byte +var z12271 [1 << 17]byte +var z12272 [1 << 17]byte +var z12273 [1 << 17]byte +var z12274 [1 << 17]byte +var z12275 [1 << 17]byte +var z12276 [1 << 17]byte +var z12277 [1 << 17]byte +var z12278 [1 << 17]byte +var z12279 [1 << 17]byte +var z12280 [1 << 17]byte +var z12281 [1 << 17]byte +var z12282 [1 << 17]byte +var z12283 [1 << 17]byte +var z12284 [1 << 17]byte +var z12285 [1 << 17]byte +var z12286 [1 << 17]byte +var z12287 [1 << 17]byte +var z12288 [1 << 17]byte +var z12289 [1 << 17]byte +var z12290 [1 << 17]byte +var z12291 [1 << 17]byte +var z12292 [1 << 17]byte +var z12293 [1 << 17]byte +var z12294 [1 << 17]byte +var z12295 [1 << 17]byte +var z12296 [1 << 17]byte +var z12297 [1 << 17]byte +var z12298 [1 << 17]byte +var z12299 [1 << 17]byte +var z12300 [1 << 17]byte +var z12301 [1 << 17]byte +var z12302 [1 << 17]byte +var z12303 [1 << 17]byte +var z12304 [1 << 17]byte +var z12305 [1 << 17]byte +var z12306 [1 << 17]byte +var z12307 [1 << 17]byte +var z12308 [1 << 17]byte +var z12309 [1 << 17]byte +var z12310 [1 << 17]byte +var z12311 [1 << 17]byte +var z12312 [1 << 17]byte +var z12313 [1 << 17]byte +var z12314 [1 << 17]byte +var z12315 [1 << 17]byte +var z12316 [1 << 17]byte +var z12317 [1 << 17]byte +var z12318 [1 << 17]byte +var z12319 [1 << 17]byte +var z12320 [1 << 17]byte +var z12321 [1 << 17]byte +var z12322 [1 << 17]byte +var z12323 [1 << 17]byte +var z12324 [1 << 17]byte +var z12325 [1 << 17]byte +var z12326 [1 << 17]byte +var z12327 [1 << 17]byte +var z12328 [1 << 17]byte +var z12329 [1 << 17]byte +var z12330 [1 << 17]byte +var z12331 [1 << 17]byte +var z12332 [1 << 17]byte +var z12333 [1 << 17]byte +var z12334 [1 << 17]byte +var z12335 [1 << 17]byte +var z12336 [1 << 17]byte +var z12337 [1 << 17]byte +var z12338 [1 << 17]byte +var z12339 [1 << 17]byte +var z12340 [1 << 17]byte +var z12341 [1 << 17]byte +var z12342 [1 << 17]byte +var z12343 [1 << 17]byte +var z12344 [1 << 17]byte +var z12345 [1 << 17]byte +var z12346 [1 << 17]byte +var z12347 [1 << 17]byte +var z12348 [1 << 17]byte +var z12349 [1 << 17]byte +var z12350 [1 << 17]byte +var z12351 [1 << 17]byte +var z12352 [1 << 17]byte +var z12353 [1 << 17]byte +var z12354 [1 << 17]byte +var z12355 [1 << 17]byte +var z12356 [1 << 17]byte +var z12357 [1 << 17]byte +var z12358 [1 << 17]byte +var z12359 [1 << 17]byte +var z12360 [1 << 17]byte +var z12361 [1 << 17]byte +var z12362 [1 << 17]byte +var z12363 [1 << 17]byte +var z12364 [1 << 17]byte +var z12365 [1 << 17]byte +var z12366 [1 << 17]byte +var z12367 [1 << 17]byte +var z12368 [1 << 17]byte +var z12369 [1 << 17]byte +var z12370 [1 << 17]byte +var z12371 [1 << 17]byte +var z12372 [1 << 17]byte +var z12373 [1 << 17]byte +var z12374 [1 << 17]byte +var z12375 [1 << 17]byte +var z12376 [1 << 17]byte +var z12377 [1 << 17]byte +var z12378 [1 << 17]byte +var z12379 [1 << 17]byte +var z12380 [1 << 17]byte +var z12381 [1 << 17]byte +var z12382 [1 << 17]byte +var z12383 [1 << 17]byte +var z12384 [1 << 17]byte +var z12385 [1 << 17]byte +var z12386 [1 << 17]byte +var z12387 [1 << 17]byte +var z12388 [1 << 17]byte +var z12389 [1 << 17]byte +var z12390 [1 << 17]byte +var z12391 [1 << 17]byte +var z12392 [1 << 17]byte +var z12393 [1 << 17]byte +var z12394 [1 << 17]byte +var z12395 [1 << 17]byte +var z12396 [1 << 17]byte +var z12397 [1 << 17]byte +var z12398 [1 << 17]byte +var z12399 [1 << 17]byte +var z12400 [1 << 17]byte +var z12401 [1 << 17]byte +var z12402 [1 << 17]byte +var z12403 [1 << 17]byte +var z12404 [1 << 17]byte +var z12405 [1 << 17]byte +var z12406 [1 << 17]byte +var z12407 [1 << 17]byte +var z12408 [1 << 17]byte +var z12409 [1 << 17]byte +var z12410 [1 << 17]byte +var z12411 [1 << 17]byte +var z12412 [1 << 17]byte +var z12413 [1 << 17]byte +var z12414 [1 << 17]byte +var z12415 [1 << 17]byte +var z12416 [1 << 17]byte +var z12417 [1 << 17]byte +var z12418 [1 << 17]byte +var z12419 [1 << 17]byte +var z12420 [1 << 17]byte +var z12421 [1 << 17]byte +var z12422 [1 << 17]byte +var z12423 [1 << 17]byte +var z12424 [1 << 17]byte +var z12425 [1 << 17]byte +var z12426 [1 << 17]byte +var z12427 [1 << 17]byte +var z12428 [1 << 17]byte +var z12429 [1 << 17]byte +var z12430 [1 << 17]byte +var z12431 [1 << 17]byte +var z12432 [1 << 17]byte +var z12433 [1 << 17]byte +var z12434 [1 << 17]byte +var z12435 [1 << 17]byte +var z12436 [1 << 17]byte +var z12437 [1 << 17]byte +var z12438 [1 << 17]byte +var z12439 [1 << 17]byte +var z12440 [1 << 17]byte +var z12441 [1 << 17]byte +var z12442 [1 << 17]byte +var z12443 [1 << 17]byte +var z12444 [1 << 17]byte +var z12445 [1 << 17]byte +var z12446 [1 << 17]byte +var z12447 [1 << 17]byte +var z12448 [1 << 17]byte +var z12449 [1 << 17]byte +var z12450 [1 << 17]byte +var z12451 [1 << 17]byte +var z12452 [1 << 17]byte +var z12453 [1 << 17]byte +var z12454 [1 << 17]byte +var z12455 [1 << 17]byte +var z12456 [1 << 17]byte +var z12457 [1 << 17]byte +var z12458 [1 << 17]byte +var z12459 [1 << 17]byte +var z12460 [1 << 17]byte +var z12461 [1 << 17]byte +var z12462 [1 << 17]byte +var z12463 [1 << 17]byte +var z12464 [1 << 17]byte +var z12465 [1 << 17]byte +var z12466 [1 << 17]byte +var z12467 [1 << 17]byte +var z12468 [1 << 17]byte +var z12469 [1 << 17]byte +var z12470 [1 << 17]byte +var z12471 [1 << 17]byte +var z12472 [1 << 17]byte +var z12473 [1 << 17]byte +var z12474 [1 << 17]byte +var z12475 [1 << 17]byte +var z12476 [1 << 17]byte +var z12477 [1 << 17]byte +var z12478 [1 << 17]byte +var z12479 [1 << 17]byte +var z12480 [1 << 17]byte +var z12481 [1 << 17]byte +var z12482 [1 << 17]byte +var z12483 [1 << 17]byte +var z12484 [1 << 17]byte +var z12485 [1 << 17]byte +var z12486 [1 << 17]byte +var z12487 [1 << 17]byte +var z12488 [1 << 17]byte +var z12489 [1 << 17]byte +var z12490 [1 << 17]byte +var z12491 [1 << 17]byte +var z12492 [1 << 17]byte +var z12493 [1 << 17]byte +var z12494 [1 << 17]byte +var z12495 [1 << 17]byte +var z12496 [1 << 17]byte +var z12497 [1 << 17]byte +var z12498 [1 << 17]byte +var z12499 [1 << 17]byte +var z12500 [1 << 17]byte +var z12501 [1 << 17]byte +var z12502 [1 << 17]byte +var z12503 [1 << 17]byte +var z12504 [1 << 17]byte +var z12505 [1 << 17]byte +var z12506 [1 << 17]byte +var z12507 [1 << 17]byte +var z12508 [1 << 17]byte +var z12509 [1 << 17]byte +var z12510 [1 << 17]byte +var z12511 [1 << 17]byte +var z12512 [1 << 17]byte +var z12513 [1 << 17]byte +var z12514 [1 << 17]byte +var z12515 [1 << 17]byte +var z12516 [1 << 17]byte +var z12517 [1 << 17]byte +var z12518 [1 << 17]byte +var z12519 [1 << 17]byte +var z12520 [1 << 17]byte +var z12521 [1 << 17]byte +var z12522 [1 << 17]byte +var z12523 [1 << 17]byte +var z12524 [1 << 17]byte +var z12525 [1 << 17]byte +var z12526 [1 << 17]byte +var z12527 [1 << 17]byte +var z12528 [1 << 17]byte +var z12529 [1 << 17]byte +var z12530 [1 << 17]byte +var z12531 [1 << 17]byte +var z12532 [1 << 17]byte +var z12533 [1 << 17]byte +var z12534 [1 << 17]byte +var z12535 [1 << 17]byte +var z12536 [1 << 17]byte +var z12537 [1 << 17]byte +var z12538 [1 << 17]byte +var z12539 [1 << 17]byte +var z12540 [1 << 17]byte +var z12541 [1 << 17]byte +var z12542 [1 << 17]byte +var z12543 [1 << 17]byte +var z12544 [1 << 17]byte +var z12545 [1 << 17]byte +var z12546 [1 << 17]byte +var z12547 [1 << 17]byte +var z12548 [1 << 17]byte +var z12549 [1 << 17]byte +var z12550 [1 << 17]byte +var z12551 [1 << 17]byte +var z12552 [1 << 17]byte +var z12553 [1 << 17]byte +var z12554 [1 << 17]byte +var z12555 [1 << 17]byte +var z12556 [1 << 17]byte +var z12557 [1 << 17]byte +var z12558 [1 << 17]byte +var z12559 [1 << 17]byte +var z12560 [1 << 17]byte +var z12561 [1 << 17]byte +var z12562 [1 << 17]byte +var z12563 [1 << 17]byte +var z12564 [1 << 17]byte +var z12565 [1 << 17]byte +var z12566 [1 << 17]byte +var z12567 [1 << 17]byte +var z12568 [1 << 17]byte +var z12569 [1 << 17]byte +var z12570 [1 << 17]byte +var z12571 [1 << 17]byte +var z12572 [1 << 17]byte +var z12573 [1 << 17]byte +var z12574 [1 << 17]byte +var z12575 [1 << 17]byte +var z12576 [1 << 17]byte +var z12577 [1 << 17]byte +var z12578 [1 << 17]byte +var z12579 [1 << 17]byte +var z12580 [1 << 17]byte +var z12581 [1 << 17]byte +var z12582 [1 << 17]byte +var z12583 [1 << 17]byte +var z12584 [1 << 17]byte +var z12585 [1 << 17]byte +var z12586 [1 << 17]byte +var z12587 [1 << 17]byte +var z12588 [1 << 17]byte +var z12589 [1 << 17]byte +var z12590 [1 << 17]byte +var z12591 [1 << 17]byte +var z12592 [1 << 17]byte +var z12593 [1 << 17]byte +var z12594 [1 << 17]byte +var z12595 [1 << 17]byte +var z12596 [1 << 17]byte +var z12597 [1 << 17]byte +var z12598 [1 << 17]byte +var z12599 [1 << 17]byte +var z12600 [1 << 17]byte +var z12601 [1 << 17]byte +var z12602 [1 << 17]byte +var z12603 [1 << 17]byte +var z12604 [1 << 17]byte +var z12605 [1 << 17]byte +var z12606 [1 << 17]byte +var z12607 [1 << 17]byte +var z12608 [1 << 17]byte +var z12609 [1 << 17]byte +var z12610 [1 << 17]byte +var z12611 [1 << 17]byte +var z12612 [1 << 17]byte +var z12613 [1 << 17]byte +var z12614 [1 << 17]byte +var z12615 [1 << 17]byte +var z12616 [1 << 17]byte +var z12617 [1 << 17]byte +var z12618 [1 << 17]byte +var z12619 [1 << 17]byte +var z12620 [1 << 17]byte +var z12621 [1 << 17]byte +var z12622 [1 << 17]byte +var z12623 [1 << 17]byte +var z12624 [1 << 17]byte +var z12625 [1 << 17]byte +var z12626 [1 << 17]byte +var z12627 [1 << 17]byte +var z12628 [1 << 17]byte +var z12629 [1 << 17]byte +var z12630 [1 << 17]byte +var z12631 [1 << 17]byte +var z12632 [1 << 17]byte +var z12633 [1 << 17]byte +var z12634 [1 << 17]byte +var z12635 [1 << 17]byte +var z12636 [1 << 17]byte +var z12637 [1 << 17]byte +var z12638 [1 << 17]byte +var z12639 [1 << 17]byte +var z12640 [1 << 17]byte +var z12641 [1 << 17]byte +var z12642 [1 << 17]byte +var z12643 [1 << 17]byte +var z12644 [1 << 17]byte +var z12645 [1 << 17]byte +var z12646 [1 << 17]byte +var z12647 [1 << 17]byte +var z12648 [1 << 17]byte +var z12649 [1 << 17]byte +var z12650 [1 << 17]byte +var z12651 [1 << 17]byte +var z12652 [1 << 17]byte +var z12653 [1 << 17]byte +var z12654 [1 << 17]byte +var z12655 [1 << 17]byte +var z12656 [1 << 17]byte +var z12657 [1 << 17]byte +var z12658 [1 << 17]byte +var z12659 [1 << 17]byte +var z12660 [1 << 17]byte +var z12661 [1 << 17]byte +var z12662 [1 << 17]byte +var z12663 [1 << 17]byte +var z12664 [1 << 17]byte +var z12665 [1 << 17]byte +var z12666 [1 << 17]byte +var z12667 [1 << 17]byte +var z12668 [1 << 17]byte +var z12669 [1 << 17]byte +var z12670 [1 << 17]byte +var z12671 [1 << 17]byte +var z12672 [1 << 17]byte +var z12673 [1 << 17]byte +var z12674 [1 << 17]byte +var z12675 [1 << 17]byte +var z12676 [1 << 17]byte +var z12677 [1 << 17]byte +var z12678 [1 << 17]byte +var z12679 [1 << 17]byte +var z12680 [1 << 17]byte +var z12681 [1 << 17]byte +var z12682 [1 << 17]byte +var z12683 [1 << 17]byte +var z12684 [1 << 17]byte +var z12685 [1 << 17]byte +var z12686 [1 << 17]byte +var z12687 [1 << 17]byte +var z12688 [1 << 17]byte +var z12689 [1 << 17]byte +var z12690 [1 << 17]byte +var z12691 [1 << 17]byte +var z12692 [1 << 17]byte +var z12693 [1 << 17]byte +var z12694 [1 << 17]byte +var z12695 [1 << 17]byte +var z12696 [1 << 17]byte +var z12697 [1 << 17]byte +var z12698 [1 << 17]byte +var z12699 [1 << 17]byte +var z12700 [1 << 17]byte +var z12701 [1 << 17]byte +var z12702 [1 << 17]byte +var z12703 [1 << 17]byte +var z12704 [1 << 17]byte +var z12705 [1 << 17]byte +var z12706 [1 << 17]byte +var z12707 [1 << 17]byte +var z12708 [1 << 17]byte +var z12709 [1 << 17]byte +var z12710 [1 << 17]byte +var z12711 [1 << 17]byte +var z12712 [1 << 17]byte +var z12713 [1 << 17]byte +var z12714 [1 << 17]byte +var z12715 [1 << 17]byte +var z12716 [1 << 17]byte +var z12717 [1 << 17]byte +var z12718 [1 << 17]byte +var z12719 [1 << 17]byte +var z12720 [1 << 17]byte +var z12721 [1 << 17]byte +var z12722 [1 << 17]byte +var z12723 [1 << 17]byte +var z12724 [1 << 17]byte +var z12725 [1 << 17]byte +var z12726 [1 << 17]byte +var z12727 [1 << 17]byte +var z12728 [1 << 17]byte +var z12729 [1 << 17]byte +var z12730 [1 << 17]byte +var z12731 [1 << 17]byte +var z12732 [1 << 17]byte +var z12733 [1 << 17]byte +var z12734 [1 << 17]byte +var z12735 [1 << 17]byte +var z12736 [1 << 17]byte +var z12737 [1 << 17]byte +var z12738 [1 << 17]byte +var z12739 [1 << 17]byte +var z12740 [1 << 17]byte +var z12741 [1 << 17]byte +var z12742 [1 << 17]byte +var z12743 [1 << 17]byte +var z12744 [1 << 17]byte +var z12745 [1 << 17]byte +var z12746 [1 << 17]byte +var z12747 [1 << 17]byte +var z12748 [1 << 17]byte +var z12749 [1 << 17]byte +var z12750 [1 << 17]byte +var z12751 [1 << 17]byte +var z12752 [1 << 17]byte +var z12753 [1 << 17]byte +var z12754 [1 << 17]byte +var z12755 [1 << 17]byte +var z12756 [1 << 17]byte +var z12757 [1 << 17]byte +var z12758 [1 << 17]byte +var z12759 [1 << 17]byte +var z12760 [1 << 17]byte +var z12761 [1 << 17]byte +var z12762 [1 << 17]byte +var z12763 [1 << 17]byte +var z12764 [1 << 17]byte +var z12765 [1 << 17]byte +var z12766 [1 << 17]byte +var z12767 [1 << 17]byte +var z12768 [1 << 17]byte +var z12769 [1 << 17]byte +var z12770 [1 << 17]byte +var z12771 [1 << 17]byte +var z12772 [1 << 17]byte +var z12773 [1 << 17]byte +var z12774 [1 << 17]byte +var z12775 [1 << 17]byte +var z12776 [1 << 17]byte +var z12777 [1 << 17]byte +var z12778 [1 << 17]byte +var z12779 [1 << 17]byte +var z12780 [1 << 17]byte +var z12781 [1 << 17]byte +var z12782 [1 << 17]byte +var z12783 [1 << 17]byte +var z12784 [1 << 17]byte +var z12785 [1 << 17]byte +var z12786 [1 << 17]byte +var z12787 [1 << 17]byte +var z12788 [1 << 17]byte +var z12789 [1 << 17]byte +var z12790 [1 << 17]byte +var z12791 [1 << 17]byte +var z12792 [1 << 17]byte +var z12793 [1 << 17]byte +var z12794 [1 << 17]byte +var z12795 [1 << 17]byte +var z12796 [1 << 17]byte +var z12797 [1 << 17]byte +var z12798 [1 << 17]byte +var z12799 [1 << 17]byte +var z12800 [1 << 17]byte +var z12801 [1 << 17]byte +var z12802 [1 << 17]byte +var z12803 [1 << 17]byte +var z12804 [1 << 17]byte +var z12805 [1 << 17]byte +var z12806 [1 << 17]byte +var z12807 [1 << 17]byte +var z12808 [1 << 17]byte +var z12809 [1 << 17]byte +var z12810 [1 << 17]byte +var z12811 [1 << 17]byte +var z12812 [1 << 17]byte +var z12813 [1 << 17]byte +var z12814 [1 << 17]byte +var z12815 [1 << 17]byte +var z12816 [1 << 17]byte +var z12817 [1 << 17]byte +var z12818 [1 << 17]byte +var z12819 [1 << 17]byte +var z12820 [1 << 17]byte +var z12821 [1 << 17]byte +var z12822 [1 << 17]byte +var z12823 [1 << 17]byte +var z12824 [1 << 17]byte +var z12825 [1 << 17]byte +var z12826 [1 << 17]byte +var z12827 [1 << 17]byte +var z12828 [1 << 17]byte +var z12829 [1 << 17]byte +var z12830 [1 << 17]byte +var z12831 [1 << 17]byte +var z12832 [1 << 17]byte +var z12833 [1 << 17]byte +var z12834 [1 << 17]byte +var z12835 [1 << 17]byte +var z12836 [1 << 17]byte +var z12837 [1 << 17]byte +var z12838 [1 << 17]byte +var z12839 [1 << 17]byte +var z12840 [1 << 17]byte +var z12841 [1 << 17]byte +var z12842 [1 << 17]byte +var z12843 [1 << 17]byte +var z12844 [1 << 17]byte +var z12845 [1 << 17]byte +var z12846 [1 << 17]byte +var z12847 [1 << 17]byte +var z12848 [1 << 17]byte +var z12849 [1 << 17]byte +var z12850 [1 << 17]byte +var z12851 [1 << 17]byte +var z12852 [1 << 17]byte +var z12853 [1 << 17]byte +var z12854 [1 << 17]byte +var z12855 [1 << 17]byte +var z12856 [1 << 17]byte +var z12857 [1 << 17]byte +var z12858 [1 << 17]byte +var z12859 [1 << 17]byte +var z12860 [1 << 17]byte +var z12861 [1 << 17]byte +var z12862 [1 << 17]byte +var z12863 [1 << 17]byte +var z12864 [1 << 17]byte +var z12865 [1 << 17]byte +var z12866 [1 << 17]byte +var z12867 [1 << 17]byte +var z12868 [1 << 17]byte +var z12869 [1 << 17]byte +var z12870 [1 << 17]byte +var z12871 [1 << 17]byte +var z12872 [1 << 17]byte +var z12873 [1 << 17]byte +var z12874 [1 << 17]byte +var z12875 [1 << 17]byte +var z12876 [1 << 17]byte +var z12877 [1 << 17]byte +var z12878 [1 << 17]byte +var z12879 [1 << 17]byte +var z12880 [1 << 17]byte +var z12881 [1 << 17]byte +var z12882 [1 << 17]byte +var z12883 [1 << 17]byte +var z12884 [1 << 17]byte +var z12885 [1 << 17]byte +var z12886 [1 << 17]byte +var z12887 [1 << 17]byte +var z12888 [1 << 17]byte +var z12889 [1 << 17]byte +var z12890 [1 << 17]byte +var z12891 [1 << 17]byte +var z12892 [1 << 17]byte +var z12893 [1 << 17]byte +var z12894 [1 << 17]byte +var z12895 [1 << 17]byte +var z12896 [1 << 17]byte +var z12897 [1 << 17]byte +var z12898 [1 << 17]byte +var z12899 [1 << 17]byte +var z12900 [1 << 17]byte +var z12901 [1 << 17]byte +var z12902 [1 << 17]byte +var z12903 [1 << 17]byte +var z12904 [1 << 17]byte +var z12905 [1 << 17]byte +var z12906 [1 << 17]byte +var z12907 [1 << 17]byte +var z12908 [1 << 17]byte +var z12909 [1 << 17]byte +var z12910 [1 << 17]byte +var z12911 [1 << 17]byte +var z12912 [1 << 17]byte +var z12913 [1 << 17]byte +var z12914 [1 << 17]byte +var z12915 [1 << 17]byte +var z12916 [1 << 17]byte +var z12917 [1 << 17]byte +var z12918 [1 << 17]byte +var z12919 [1 << 17]byte +var z12920 [1 << 17]byte +var z12921 [1 << 17]byte +var z12922 [1 << 17]byte +var z12923 [1 << 17]byte +var z12924 [1 << 17]byte +var z12925 [1 << 17]byte +var z12926 [1 << 17]byte +var z12927 [1 << 17]byte +var z12928 [1 << 17]byte +var z12929 [1 << 17]byte +var z12930 [1 << 17]byte +var z12931 [1 << 17]byte +var z12932 [1 << 17]byte +var z12933 [1 << 17]byte +var z12934 [1 << 17]byte +var z12935 [1 << 17]byte +var z12936 [1 << 17]byte +var z12937 [1 << 17]byte +var z12938 [1 << 17]byte +var z12939 [1 << 17]byte +var z12940 [1 << 17]byte +var z12941 [1 << 17]byte +var z12942 [1 << 17]byte +var z12943 [1 << 17]byte +var z12944 [1 << 17]byte +var z12945 [1 << 17]byte +var z12946 [1 << 17]byte +var z12947 [1 << 17]byte +var z12948 [1 << 17]byte +var z12949 [1 << 17]byte +var z12950 [1 << 17]byte +var z12951 [1 << 17]byte +var z12952 [1 << 17]byte +var z12953 [1 << 17]byte +var z12954 [1 << 17]byte +var z12955 [1 << 17]byte +var z12956 [1 << 17]byte +var z12957 [1 << 17]byte +var z12958 [1 << 17]byte +var z12959 [1 << 17]byte +var z12960 [1 << 17]byte +var z12961 [1 << 17]byte +var z12962 [1 << 17]byte +var z12963 [1 << 17]byte +var z12964 [1 << 17]byte +var z12965 [1 << 17]byte +var z12966 [1 << 17]byte +var z12967 [1 << 17]byte +var z12968 [1 << 17]byte +var z12969 [1 << 17]byte +var z12970 [1 << 17]byte +var z12971 [1 << 17]byte +var z12972 [1 << 17]byte +var z12973 [1 << 17]byte +var z12974 [1 << 17]byte +var z12975 [1 << 17]byte +var z12976 [1 << 17]byte +var z12977 [1 << 17]byte +var z12978 [1 << 17]byte +var z12979 [1 << 17]byte +var z12980 [1 << 17]byte +var z12981 [1 << 17]byte +var z12982 [1 << 17]byte +var z12983 [1 << 17]byte +var z12984 [1 << 17]byte +var z12985 [1 << 17]byte +var z12986 [1 << 17]byte +var z12987 [1 << 17]byte +var z12988 [1 << 17]byte +var z12989 [1 << 17]byte +var z12990 [1 << 17]byte +var z12991 [1 << 17]byte +var z12992 [1 << 17]byte +var z12993 [1 << 17]byte +var z12994 [1 << 17]byte +var z12995 [1 << 17]byte +var z12996 [1 << 17]byte +var z12997 [1 << 17]byte +var z12998 [1 << 17]byte +var z12999 [1 << 17]byte +var z13000 [1 << 17]byte +var z13001 [1 << 17]byte +var z13002 [1 << 17]byte +var z13003 [1 << 17]byte +var z13004 [1 << 17]byte +var z13005 [1 << 17]byte +var z13006 [1 << 17]byte +var z13007 [1 << 17]byte +var z13008 [1 << 17]byte +var z13009 [1 << 17]byte +var z13010 [1 << 17]byte +var z13011 [1 << 17]byte +var z13012 [1 << 17]byte +var z13013 [1 << 17]byte +var z13014 [1 << 17]byte +var z13015 [1 << 17]byte +var z13016 [1 << 17]byte +var z13017 [1 << 17]byte +var z13018 [1 << 17]byte +var z13019 [1 << 17]byte +var z13020 [1 << 17]byte +var z13021 [1 << 17]byte +var z13022 [1 << 17]byte +var z13023 [1 << 17]byte +var z13024 [1 << 17]byte +var z13025 [1 << 17]byte +var z13026 [1 << 17]byte +var z13027 [1 << 17]byte +var z13028 [1 << 17]byte +var z13029 [1 << 17]byte +var z13030 [1 << 17]byte +var z13031 [1 << 17]byte +var z13032 [1 << 17]byte +var z13033 [1 << 17]byte +var z13034 [1 << 17]byte +var z13035 [1 << 17]byte +var z13036 [1 << 17]byte +var z13037 [1 << 17]byte +var z13038 [1 << 17]byte +var z13039 [1 << 17]byte +var z13040 [1 << 17]byte +var z13041 [1 << 17]byte +var z13042 [1 << 17]byte +var z13043 [1 << 17]byte +var z13044 [1 << 17]byte +var z13045 [1 << 17]byte +var z13046 [1 << 17]byte +var z13047 [1 << 17]byte +var z13048 [1 << 17]byte +var z13049 [1 << 17]byte +var z13050 [1 << 17]byte +var z13051 [1 << 17]byte +var z13052 [1 << 17]byte +var z13053 [1 << 17]byte +var z13054 [1 << 17]byte +var z13055 [1 << 17]byte +var z13056 [1 << 17]byte +var z13057 [1 << 17]byte +var z13058 [1 << 17]byte +var z13059 [1 << 17]byte +var z13060 [1 << 17]byte +var z13061 [1 << 17]byte +var z13062 [1 << 17]byte +var z13063 [1 << 17]byte +var z13064 [1 << 17]byte +var z13065 [1 << 17]byte +var z13066 [1 << 17]byte +var z13067 [1 << 17]byte +var z13068 [1 << 17]byte +var z13069 [1 << 17]byte +var z13070 [1 << 17]byte +var z13071 [1 << 17]byte +var z13072 [1 << 17]byte +var z13073 [1 << 17]byte +var z13074 [1 << 17]byte +var z13075 [1 << 17]byte +var z13076 [1 << 17]byte +var z13077 [1 << 17]byte +var z13078 [1 << 17]byte +var z13079 [1 << 17]byte +var z13080 [1 << 17]byte +var z13081 [1 << 17]byte +var z13082 [1 << 17]byte +var z13083 [1 << 17]byte +var z13084 [1 << 17]byte +var z13085 [1 << 17]byte +var z13086 [1 << 17]byte +var z13087 [1 << 17]byte +var z13088 [1 << 17]byte +var z13089 [1 << 17]byte +var z13090 [1 << 17]byte +var z13091 [1 << 17]byte +var z13092 [1 << 17]byte +var z13093 [1 << 17]byte +var z13094 [1 << 17]byte +var z13095 [1 << 17]byte +var z13096 [1 << 17]byte +var z13097 [1 << 17]byte +var z13098 [1 << 17]byte +var z13099 [1 << 17]byte +var z13100 [1 << 17]byte +var z13101 [1 << 17]byte +var z13102 [1 << 17]byte +var z13103 [1 << 17]byte +var z13104 [1 << 17]byte +var z13105 [1 << 17]byte +var z13106 [1 << 17]byte +var z13107 [1 << 17]byte +var z13108 [1 << 17]byte +var z13109 [1 << 17]byte +var z13110 [1 << 17]byte +var z13111 [1 << 17]byte +var z13112 [1 << 17]byte +var z13113 [1 << 17]byte +var z13114 [1 << 17]byte +var z13115 [1 << 17]byte +var z13116 [1 << 17]byte +var z13117 [1 << 17]byte +var z13118 [1 << 17]byte +var z13119 [1 << 17]byte +var z13120 [1 << 17]byte +var z13121 [1 << 17]byte +var z13122 [1 << 17]byte +var z13123 [1 << 17]byte +var z13124 [1 << 17]byte +var z13125 [1 << 17]byte +var z13126 [1 << 17]byte +var z13127 [1 << 17]byte +var z13128 [1 << 17]byte +var z13129 [1 << 17]byte +var z13130 [1 << 17]byte +var z13131 [1 << 17]byte +var z13132 [1 << 17]byte +var z13133 [1 << 17]byte +var z13134 [1 << 17]byte +var z13135 [1 << 17]byte +var z13136 [1 << 17]byte +var z13137 [1 << 17]byte +var z13138 [1 << 17]byte +var z13139 [1 << 17]byte +var z13140 [1 << 17]byte +var z13141 [1 << 17]byte +var z13142 [1 << 17]byte +var z13143 [1 << 17]byte +var z13144 [1 << 17]byte +var z13145 [1 << 17]byte +var z13146 [1 << 17]byte +var z13147 [1 << 17]byte +var z13148 [1 << 17]byte +var z13149 [1 << 17]byte +var z13150 [1 << 17]byte +var z13151 [1 << 17]byte +var z13152 [1 << 17]byte +var z13153 [1 << 17]byte +var z13154 [1 << 17]byte +var z13155 [1 << 17]byte +var z13156 [1 << 17]byte +var z13157 [1 << 17]byte +var z13158 [1 << 17]byte +var z13159 [1 << 17]byte +var z13160 [1 << 17]byte +var z13161 [1 << 17]byte +var z13162 [1 << 17]byte +var z13163 [1 << 17]byte +var z13164 [1 << 17]byte +var z13165 [1 << 17]byte +var z13166 [1 << 17]byte +var z13167 [1 << 17]byte +var z13168 [1 << 17]byte +var z13169 [1 << 17]byte +var z13170 [1 << 17]byte +var z13171 [1 << 17]byte +var z13172 [1 << 17]byte +var z13173 [1 << 17]byte +var z13174 [1 << 17]byte +var z13175 [1 << 17]byte +var z13176 [1 << 17]byte +var z13177 [1 << 17]byte +var z13178 [1 << 17]byte +var z13179 [1 << 17]byte +var z13180 [1 << 17]byte +var z13181 [1 << 17]byte +var z13182 [1 << 17]byte +var z13183 [1 << 17]byte +var z13184 [1 << 17]byte +var z13185 [1 << 17]byte +var z13186 [1 << 17]byte +var z13187 [1 << 17]byte +var z13188 [1 << 17]byte +var z13189 [1 << 17]byte +var z13190 [1 << 17]byte +var z13191 [1 << 17]byte +var z13192 [1 << 17]byte +var z13193 [1 << 17]byte +var z13194 [1 << 17]byte +var z13195 [1 << 17]byte +var z13196 [1 << 17]byte +var z13197 [1 << 17]byte +var z13198 [1 << 17]byte +var z13199 [1 << 17]byte +var z13200 [1 << 17]byte +var z13201 [1 << 17]byte +var z13202 [1 << 17]byte +var z13203 [1 << 17]byte +var z13204 [1 << 17]byte +var z13205 [1 << 17]byte +var z13206 [1 << 17]byte +var z13207 [1 << 17]byte +var z13208 [1 << 17]byte +var z13209 [1 << 17]byte +var z13210 [1 << 17]byte +var z13211 [1 << 17]byte +var z13212 [1 << 17]byte +var z13213 [1 << 17]byte +var z13214 [1 << 17]byte +var z13215 [1 << 17]byte +var z13216 [1 << 17]byte +var z13217 [1 << 17]byte +var z13218 [1 << 17]byte +var z13219 [1 << 17]byte +var z13220 [1 << 17]byte +var z13221 [1 << 17]byte +var z13222 [1 << 17]byte +var z13223 [1 << 17]byte +var z13224 [1 << 17]byte +var z13225 [1 << 17]byte +var z13226 [1 << 17]byte +var z13227 [1 << 17]byte +var z13228 [1 << 17]byte +var z13229 [1 << 17]byte +var z13230 [1 << 17]byte +var z13231 [1 << 17]byte +var z13232 [1 << 17]byte +var z13233 [1 << 17]byte +var z13234 [1 << 17]byte +var z13235 [1 << 17]byte +var z13236 [1 << 17]byte +var z13237 [1 << 17]byte +var z13238 [1 << 17]byte +var z13239 [1 << 17]byte +var z13240 [1 << 17]byte +var z13241 [1 << 17]byte +var z13242 [1 << 17]byte +var z13243 [1 << 17]byte +var z13244 [1 << 17]byte +var z13245 [1 << 17]byte +var z13246 [1 << 17]byte +var z13247 [1 << 17]byte +var z13248 [1 << 17]byte +var z13249 [1 << 17]byte +var z13250 [1 << 17]byte +var z13251 [1 << 17]byte +var z13252 [1 << 17]byte +var z13253 [1 << 17]byte +var z13254 [1 << 17]byte +var z13255 [1 << 17]byte +var z13256 [1 << 17]byte +var z13257 [1 << 17]byte +var z13258 [1 << 17]byte +var z13259 [1 << 17]byte +var z13260 [1 << 17]byte +var z13261 [1 << 17]byte +var z13262 [1 << 17]byte +var z13263 [1 << 17]byte +var z13264 [1 << 17]byte +var z13265 [1 << 17]byte +var z13266 [1 << 17]byte +var z13267 [1 << 17]byte +var z13268 [1 << 17]byte +var z13269 [1 << 17]byte +var z13270 [1 << 17]byte +var z13271 [1 << 17]byte +var z13272 [1 << 17]byte +var z13273 [1 << 17]byte +var z13274 [1 << 17]byte +var z13275 [1 << 17]byte +var z13276 [1 << 17]byte +var z13277 [1 << 17]byte +var z13278 [1 << 17]byte +var z13279 [1 << 17]byte +var z13280 [1 << 17]byte +var z13281 [1 << 17]byte +var z13282 [1 << 17]byte +var z13283 [1 << 17]byte +var z13284 [1 << 17]byte +var z13285 [1 << 17]byte +var z13286 [1 << 17]byte +var z13287 [1 << 17]byte +var z13288 [1 << 17]byte +var z13289 [1 << 17]byte +var z13290 [1 << 17]byte +var z13291 [1 << 17]byte +var z13292 [1 << 17]byte +var z13293 [1 << 17]byte +var z13294 [1 << 17]byte +var z13295 [1 << 17]byte +var z13296 [1 << 17]byte +var z13297 [1 << 17]byte +var z13298 [1 << 17]byte +var z13299 [1 << 17]byte +var z13300 [1 << 17]byte +var z13301 [1 << 17]byte +var z13302 [1 << 17]byte +var z13303 [1 << 17]byte +var z13304 [1 << 17]byte +var z13305 [1 << 17]byte +var z13306 [1 << 17]byte +var z13307 [1 << 17]byte +var z13308 [1 << 17]byte +var z13309 [1 << 17]byte +var z13310 [1 << 17]byte +var z13311 [1 << 17]byte +var z13312 [1 << 17]byte +var z13313 [1 << 17]byte +var z13314 [1 << 17]byte +var z13315 [1 << 17]byte +var z13316 [1 << 17]byte +var z13317 [1 << 17]byte +var z13318 [1 << 17]byte +var z13319 [1 << 17]byte +var z13320 [1 << 17]byte +var z13321 [1 << 17]byte +var z13322 [1 << 17]byte +var z13323 [1 << 17]byte +var z13324 [1 << 17]byte +var z13325 [1 << 17]byte +var z13326 [1 << 17]byte +var z13327 [1 << 17]byte +var z13328 [1 << 17]byte +var z13329 [1 << 17]byte +var z13330 [1 << 17]byte +var z13331 [1 << 17]byte +var z13332 [1 << 17]byte +var z13333 [1 << 17]byte +var z13334 [1 << 17]byte +var z13335 [1 << 17]byte +var z13336 [1 << 17]byte +var z13337 [1 << 17]byte +var z13338 [1 << 17]byte +var z13339 [1 << 17]byte +var z13340 [1 << 17]byte +var z13341 [1 << 17]byte +var z13342 [1 << 17]byte +var z13343 [1 << 17]byte +var z13344 [1 << 17]byte +var z13345 [1 << 17]byte +var z13346 [1 << 17]byte +var z13347 [1 << 17]byte +var z13348 [1 << 17]byte +var z13349 [1 << 17]byte +var z13350 [1 << 17]byte +var z13351 [1 << 17]byte +var z13352 [1 << 17]byte +var z13353 [1 << 17]byte +var z13354 [1 << 17]byte +var z13355 [1 << 17]byte +var z13356 [1 << 17]byte +var z13357 [1 << 17]byte +var z13358 [1 << 17]byte +var z13359 [1 << 17]byte +var z13360 [1 << 17]byte +var z13361 [1 << 17]byte +var z13362 [1 << 17]byte +var z13363 [1 << 17]byte +var z13364 [1 << 17]byte +var z13365 [1 << 17]byte +var z13366 [1 << 17]byte +var z13367 [1 << 17]byte +var z13368 [1 << 17]byte +var z13369 [1 << 17]byte +var z13370 [1 << 17]byte +var z13371 [1 << 17]byte +var z13372 [1 << 17]byte +var z13373 [1 << 17]byte +var z13374 [1 << 17]byte +var z13375 [1 << 17]byte +var z13376 [1 << 17]byte +var z13377 [1 << 17]byte +var z13378 [1 << 17]byte +var z13379 [1 << 17]byte +var z13380 [1 << 17]byte +var z13381 [1 << 17]byte +var z13382 [1 << 17]byte +var z13383 [1 << 17]byte +var z13384 [1 << 17]byte +var z13385 [1 << 17]byte +var z13386 [1 << 17]byte +var z13387 [1 << 17]byte +var z13388 [1 << 17]byte +var z13389 [1 << 17]byte +var z13390 [1 << 17]byte +var z13391 [1 << 17]byte +var z13392 [1 << 17]byte +var z13393 [1 << 17]byte +var z13394 [1 << 17]byte +var z13395 [1 << 17]byte +var z13396 [1 << 17]byte +var z13397 [1 << 17]byte +var z13398 [1 << 17]byte +var z13399 [1 << 17]byte +var z13400 [1 << 17]byte +var z13401 [1 << 17]byte +var z13402 [1 << 17]byte +var z13403 [1 << 17]byte +var z13404 [1 << 17]byte +var z13405 [1 << 17]byte +var z13406 [1 << 17]byte +var z13407 [1 << 17]byte +var z13408 [1 << 17]byte +var z13409 [1 << 17]byte +var z13410 [1 << 17]byte +var z13411 [1 << 17]byte +var z13412 [1 << 17]byte +var z13413 [1 << 17]byte +var z13414 [1 << 17]byte +var z13415 [1 << 17]byte +var z13416 [1 << 17]byte +var z13417 [1 << 17]byte +var z13418 [1 << 17]byte +var z13419 [1 << 17]byte +var z13420 [1 << 17]byte +var z13421 [1 << 17]byte +var z13422 [1 << 17]byte +var z13423 [1 << 17]byte +var z13424 [1 << 17]byte +var z13425 [1 << 17]byte +var z13426 [1 << 17]byte +var z13427 [1 << 17]byte +var z13428 [1 << 17]byte +var z13429 [1 << 17]byte +var z13430 [1 << 17]byte +var z13431 [1 << 17]byte +var z13432 [1 << 17]byte +var z13433 [1 << 17]byte +var z13434 [1 << 17]byte +var z13435 [1 << 17]byte +var z13436 [1 << 17]byte +var z13437 [1 << 17]byte +var z13438 [1 << 17]byte +var z13439 [1 << 17]byte +var z13440 [1 << 17]byte +var z13441 [1 << 17]byte +var z13442 [1 << 17]byte +var z13443 [1 << 17]byte +var z13444 [1 << 17]byte +var z13445 [1 << 17]byte +var z13446 [1 << 17]byte +var z13447 [1 << 17]byte +var z13448 [1 << 17]byte +var z13449 [1 << 17]byte +var z13450 [1 << 17]byte +var z13451 [1 << 17]byte +var z13452 [1 << 17]byte +var z13453 [1 << 17]byte +var z13454 [1 << 17]byte +var z13455 [1 << 17]byte +var z13456 [1 << 17]byte +var z13457 [1 << 17]byte +var z13458 [1 << 17]byte +var z13459 [1 << 17]byte +var z13460 [1 << 17]byte +var z13461 [1 << 17]byte +var z13462 [1 << 17]byte +var z13463 [1 << 17]byte +var z13464 [1 << 17]byte +var z13465 [1 << 17]byte +var z13466 [1 << 17]byte +var z13467 [1 << 17]byte +var z13468 [1 << 17]byte +var z13469 [1 << 17]byte +var z13470 [1 << 17]byte +var z13471 [1 << 17]byte +var z13472 [1 << 17]byte +var z13473 [1 << 17]byte +var z13474 [1 << 17]byte +var z13475 [1 << 17]byte +var z13476 [1 << 17]byte +var z13477 [1 << 17]byte +var z13478 [1 << 17]byte +var z13479 [1 << 17]byte +var z13480 [1 << 17]byte +var z13481 [1 << 17]byte +var z13482 [1 << 17]byte +var z13483 [1 << 17]byte +var z13484 [1 << 17]byte +var z13485 [1 << 17]byte +var z13486 [1 << 17]byte +var z13487 [1 << 17]byte +var z13488 [1 << 17]byte +var z13489 [1 << 17]byte +var z13490 [1 << 17]byte +var z13491 [1 << 17]byte +var z13492 [1 << 17]byte +var z13493 [1 << 17]byte +var z13494 [1 << 17]byte +var z13495 [1 << 17]byte +var z13496 [1 << 17]byte +var z13497 [1 << 17]byte +var z13498 [1 << 17]byte +var z13499 [1 << 17]byte +var z13500 [1 << 17]byte +var z13501 [1 << 17]byte +var z13502 [1 << 17]byte +var z13503 [1 << 17]byte +var z13504 [1 << 17]byte +var z13505 [1 << 17]byte +var z13506 [1 << 17]byte +var z13507 [1 << 17]byte +var z13508 [1 << 17]byte +var z13509 [1 << 17]byte +var z13510 [1 << 17]byte +var z13511 [1 << 17]byte +var z13512 [1 << 17]byte +var z13513 [1 << 17]byte +var z13514 [1 << 17]byte +var z13515 [1 << 17]byte +var z13516 [1 << 17]byte +var z13517 [1 << 17]byte +var z13518 [1 << 17]byte +var z13519 [1 << 17]byte +var z13520 [1 << 17]byte +var z13521 [1 << 17]byte +var z13522 [1 << 17]byte +var z13523 [1 << 17]byte +var z13524 [1 << 17]byte +var z13525 [1 << 17]byte +var z13526 [1 << 17]byte +var z13527 [1 << 17]byte +var z13528 [1 << 17]byte +var z13529 [1 << 17]byte +var z13530 [1 << 17]byte +var z13531 [1 << 17]byte +var z13532 [1 << 17]byte +var z13533 [1 << 17]byte +var z13534 [1 << 17]byte +var z13535 [1 << 17]byte +var z13536 [1 << 17]byte +var z13537 [1 << 17]byte +var z13538 [1 << 17]byte +var z13539 [1 << 17]byte +var z13540 [1 << 17]byte +var z13541 [1 << 17]byte +var z13542 [1 << 17]byte +var z13543 [1 << 17]byte +var z13544 [1 << 17]byte +var z13545 [1 << 17]byte +var z13546 [1 << 17]byte +var z13547 [1 << 17]byte +var z13548 [1 << 17]byte +var z13549 [1 << 17]byte +var z13550 [1 << 17]byte +var z13551 [1 << 17]byte +var z13552 [1 << 17]byte +var z13553 [1 << 17]byte +var z13554 [1 << 17]byte +var z13555 [1 << 17]byte +var z13556 [1 << 17]byte +var z13557 [1 << 17]byte +var z13558 [1 << 17]byte +var z13559 [1 << 17]byte +var z13560 [1 << 17]byte +var z13561 [1 << 17]byte +var z13562 [1 << 17]byte +var z13563 [1 << 17]byte +var z13564 [1 << 17]byte +var z13565 [1 << 17]byte +var z13566 [1 << 17]byte +var z13567 [1 << 17]byte +var z13568 [1 << 17]byte +var z13569 [1 << 17]byte +var z13570 [1 << 17]byte +var z13571 [1 << 17]byte +var z13572 [1 << 17]byte +var z13573 [1 << 17]byte +var z13574 [1 << 17]byte +var z13575 [1 << 17]byte +var z13576 [1 << 17]byte +var z13577 [1 << 17]byte +var z13578 [1 << 17]byte +var z13579 [1 << 17]byte +var z13580 [1 << 17]byte +var z13581 [1 << 17]byte +var z13582 [1 << 17]byte +var z13583 [1 << 17]byte +var z13584 [1 << 17]byte +var z13585 [1 << 17]byte +var z13586 [1 << 17]byte +var z13587 [1 << 17]byte +var z13588 [1 << 17]byte +var z13589 [1 << 17]byte +var z13590 [1 << 17]byte +var z13591 [1 << 17]byte +var z13592 [1 << 17]byte +var z13593 [1 << 17]byte +var z13594 [1 << 17]byte +var z13595 [1 << 17]byte +var z13596 [1 << 17]byte +var z13597 [1 << 17]byte +var z13598 [1 << 17]byte +var z13599 [1 << 17]byte +var z13600 [1 << 17]byte +var z13601 [1 << 17]byte +var z13602 [1 << 17]byte +var z13603 [1 << 17]byte +var z13604 [1 << 17]byte +var z13605 [1 << 17]byte +var z13606 [1 << 17]byte +var z13607 [1 << 17]byte +var z13608 [1 << 17]byte +var z13609 [1 << 17]byte +var z13610 [1 << 17]byte +var z13611 [1 << 17]byte +var z13612 [1 << 17]byte +var z13613 [1 << 17]byte +var z13614 [1 << 17]byte +var z13615 [1 << 17]byte +var z13616 [1 << 17]byte +var z13617 [1 << 17]byte +var z13618 [1 << 17]byte +var z13619 [1 << 17]byte +var z13620 [1 << 17]byte +var z13621 [1 << 17]byte +var z13622 [1 << 17]byte +var z13623 [1 << 17]byte +var z13624 [1 << 17]byte +var z13625 [1 << 17]byte +var z13626 [1 << 17]byte +var z13627 [1 << 17]byte +var z13628 [1 << 17]byte +var z13629 [1 << 17]byte +var z13630 [1 << 17]byte +var z13631 [1 << 17]byte +var z13632 [1 << 17]byte +var z13633 [1 << 17]byte +var z13634 [1 << 17]byte +var z13635 [1 << 17]byte +var z13636 [1 << 17]byte +var z13637 [1 << 17]byte +var z13638 [1 << 17]byte +var z13639 [1 << 17]byte +var z13640 [1 << 17]byte +var z13641 [1 << 17]byte +var z13642 [1 << 17]byte +var z13643 [1 << 17]byte +var z13644 [1 << 17]byte +var z13645 [1 << 17]byte +var z13646 [1 << 17]byte +var z13647 [1 << 17]byte +var z13648 [1 << 17]byte +var z13649 [1 << 17]byte +var z13650 [1 << 17]byte +var z13651 [1 << 17]byte +var z13652 [1 << 17]byte +var z13653 [1 << 17]byte +var z13654 [1 << 17]byte +var z13655 [1 << 17]byte +var z13656 [1 << 17]byte +var z13657 [1 << 17]byte +var z13658 [1 << 17]byte +var z13659 [1 << 17]byte +var z13660 [1 << 17]byte +var z13661 [1 << 17]byte +var z13662 [1 << 17]byte +var z13663 [1 << 17]byte +var z13664 [1 << 17]byte +var z13665 [1 << 17]byte +var z13666 [1 << 17]byte +var z13667 [1 << 17]byte +var z13668 [1 << 17]byte +var z13669 [1 << 17]byte +var z13670 [1 << 17]byte +var z13671 [1 << 17]byte +var z13672 [1 << 17]byte +var z13673 [1 << 17]byte +var z13674 [1 << 17]byte +var z13675 [1 << 17]byte +var z13676 [1 << 17]byte +var z13677 [1 << 17]byte +var z13678 [1 << 17]byte +var z13679 [1 << 17]byte +var z13680 [1 << 17]byte +var z13681 [1 << 17]byte +var z13682 [1 << 17]byte +var z13683 [1 << 17]byte +var z13684 [1 << 17]byte +var z13685 [1 << 17]byte +var z13686 [1 << 17]byte +var z13687 [1 << 17]byte +var z13688 [1 << 17]byte +var z13689 [1 << 17]byte +var z13690 [1 << 17]byte +var z13691 [1 << 17]byte +var z13692 [1 << 17]byte +var z13693 [1 << 17]byte +var z13694 [1 << 17]byte +var z13695 [1 << 17]byte +var z13696 [1 << 17]byte +var z13697 [1 << 17]byte +var z13698 [1 << 17]byte +var z13699 [1 << 17]byte +var z13700 [1 << 17]byte +var z13701 [1 << 17]byte +var z13702 [1 << 17]byte +var z13703 [1 << 17]byte +var z13704 [1 << 17]byte +var z13705 [1 << 17]byte +var z13706 [1 << 17]byte +var z13707 [1 << 17]byte +var z13708 [1 << 17]byte +var z13709 [1 << 17]byte +var z13710 [1 << 17]byte +var z13711 [1 << 17]byte +var z13712 [1 << 17]byte +var z13713 [1 << 17]byte +var z13714 [1 << 17]byte +var z13715 [1 << 17]byte +var z13716 [1 << 17]byte +var z13717 [1 << 17]byte +var z13718 [1 << 17]byte +var z13719 [1 << 17]byte +var z13720 [1 << 17]byte +var z13721 [1 << 17]byte +var z13722 [1 << 17]byte +var z13723 [1 << 17]byte +var z13724 [1 << 17]byte +var z13725 [1 << 17]byte +var z13726 [1 << 17]byte +var z13727 [1 << 17]byte +var z13728 [1 << 17]byte +var z13729 [1 << 17]byte +var z13730 [1 << 17]byte +var z13731 [1 << 17]byte +var z13732 [1 << 17]byte +var z13733 [1 << 17]byte +var z13734 [1 << 17]byte +var z13735 [1 << 17]byte +var z13736 [1 << 17]byte +var z13737 [1 << 17]byte +var z13738 [1 << 17]byte +var z13739 [1 << 17]byte +var z13740 [1 << 17]byte +var z13741 [1 << 17]byte +var z13742 [1 << 17]byte +var z13743 [1 << 17]byte +var z13744 [1 << 17]byte +var z13745 [1 << 17]byte +var z13746 [1 << 17]byte +var z13747 [1 << 17]byte +var z13748 [1 << 17]byte +var z13749 [1 << 17]byte +var z13750 [1 << 17]byte +var z13751 [1 << 17]byte +var z13752 [1 << 17]byte +var z13753 [1 << 17]byte +var z13754 [1 << 17]byte +var z13755 [1 << 17]byte +var z13756 [1 << 17]byte +var z13757 [1 << 17]byte +var z13758 [1 << 17]byte +var z13759 [1 << 17]byte +var z13760 [1 << 17]byte +var z13761 [1 << 17]byte +var z13762 [1 << 17]byte +var z13763 [1 << 17]byte +var z13764 [1 << 17]byte +var z13765 [1 << 17]byte +var z13766 [1 << 17]byte +var z13767 [1 << 17]byte +var z13768 [1 << 17]byte +var z13769 [1 << 17]byte +var z13770 [1 << 17]byte +var z13771 [1 << 17]byte +var z13772 [1 << 17]byte +var z13773 [1 << 17]byte +var z13774 [1 << 17]byte +var z13775 [1 << 17]byte +var z13776 [1 << 17]byte +var z13777 [1 << 17]byte +var z13778 [1 << 17]byte +var z13779 [1 << 17]byte +var z13780 [1 << 17]byte +var z13781 [1 << 17]byte +var z13782 [1 << 17]byte +var z13783 [1 << 17]byte +var z13784 [1 << 17]byte +var z13785 [1 << 17]byte +var z13786 [1 << 17]byte +var z13787 [1 << 17]byte +var z13788 [1 << 17]byte +var z13789 [1 << 17]byte +var z13790 [1 << 17]byte +var z13791 [1 << 17]byte +var z13792 [1 << 17]byte +var z13793 [1 << 17]byte +var z13794 [1 << 17]byte +var z13795 [1 << 17]byte +var z13796 [1 << 17]byte +var z13797 [1 << 17]byte +var z13798 [1 << 17]byte +var z13799 [1 << 17]byte +var z13800 [1 << 17]byte +var z13801 [1 << 17]byte +var z13802 [1 << 17]byte +var z13803 [1 << 17]byte +var z13804 [1 << 17]byte +var z13805 [1 << 17]byte +var z13806 [1 << 17]byte +var z13807 [1 << 17]byte +var z13808 [1 << 17]byte +var z13809 [1 << 17]byte +var z13810 [1 << 17]byte +var z13811 [1 << 17]byte +var z13812 [1 << 17]byte +var z13813 [1 << 17]byte +var z13814 [1 << 17]byte +var z13815 [1 << 17]byte +var z13816 [1 << 17]byte +var z13817 [1 << 17]byte +var z13818 [1 << 17]byte +var z13819 [1 << 17]byte +var z13820 [1 << 17]byte +var z13821 [1 << 17]byte +var z13822 [1 << 17]byte +var z13823 [1 << 17]byte +var z13824 [1 << 17]byte +var z13825 [1 << 17]byte +var z13826 [1 << 17]byte +var z13827 [1 << 17]byte +var z13828 [1 << 17]byte +var z13829 [1 << 17]byte +var z13830 [1 << 17]byte +var z13831 [1 << 17]byte +var z13832 [1 << 17]byte +var z13833 [1 << 17]byte +var z13834 [1 << 17]byte +var z13835 [1 << 17]byte +var z13836 [1 << 17]byte +var z13837 [1 << 17]byte +var z13838 [1 << 17]byte +var z13839 [1 << 17]byte +var z13840 [1 << 17]byte +var z13841 [1 << 17]byte +var z13842 [1 << 17]byte +var z13843 [1 << 17]byte +var z13844 [1 << 17]byte +var z13845 [1 << 17]byte +var z13846 [1 << 17]byte +var z13847 [1 << 17]byte +var z13848 [1 << 17]byte +var z13849 [1 << 17]byte +var z13850 [1 << 17]byte +var z13851 [1 << 17]byte +var z13852 [1 << 17]byte +var z13853 [1 << 17]byte +var z13854 [1 << 17]byte +var z13855 [1 << 17]byte +var z13856 [1 << 17]byte +var z13857 [1 << 17]byte +var z13858 [1 << 17]byte +var z13859 [1 << 17]byte +var z13860 [1 << 17]byte +var z13861 [1 << 17]byte +var z13862 [1 << 17]byte +var z13863 [1 << 17]byte +var z13864 [1 << 17]byte +var z13865 [1 << 17]byte +var z13866 [1 << 17]byte +var z13867 [1 << 17]byte +var z13868 [1 << 17]byte +var z13869 [1 << 17]byte +var z13870 [1 << 17]byte +var z13871 [1 << 17]byte +var z13872 [1 << 17]byte +var z13873 [1 << 17]byte +var z13874 [1 << 17]byte +var z13875 [1 << 17]byte +var z13876 [1 << 17]byte +var z13877 [1 << 17]byte +var z13878 [1 << 17]byte +var z13879 [1 << 17]byte +var z13880 [1 << 17]byte +var z13881 [1 << 17]byte +var z13882 [1 << 17]byte +var z13883 [1 << 17]byte +var z13884 [1 << 17]byte +var z13885 [1 << 17]byte +var z13886 [1 << 17]byte +var z13887 [1 << 17]byte +var z13888 [1 << 17]byte +var z13889 [1 << 17]byte +var z13890 [1 << 17]byte +var z13891 [1 << 17]byte +var z13892 [1 << 17]byte +var z13893 [1 << 17]byte +var z13894 [1 << 17]byte +var z13895 [1 << 17]byte +var z13896 [1 << 17]byte +var z13897 [1 << 17]byte +var z13898 [1 << 17]byte +var z13899 [1 << 17]byte +var z13900 [1 << 17]byte +var z13901 [1 << 17]byte +var z13902 [1 << 17]byte +var z13903 [1 << 17]byte +var z13904 [1 << 17]byte +var z13905 [1 << 17]byte +var z13906 [1 << 17]byte +var z13907 [1 << 17]byte +var z13908 [1 << 17]byte +var z13909 [1 << 17]byte +var z13910 [1 << 17]byte +var z13911 [1 << 17]byte +var z13912 [1 << 17]byte +var z13913 [1 << 17]byte +var z13914 [1 << 17]byte +var z13915 [1 << 17]byte +var z13916 [1 << 17]byte +var z13917 [1 << 17]byte +var z13918 [1 << 17]byte +var z13919 [1 << 17]byte +var z13920 [1 << 17]byte +var z13921 [1 << 17]byte +var z13922 [1 << 17]byte +var z13923 [1 << 17]byte +var z13924 [1 << 17]byte +var z13925 [1 << 17]byte +var z13926 [1 << 17]byte +var z13927 [1 << 17]byte +var z13928 [1 << 17]byte +var z13929 [1 << 17]byte +var z13930 [1 << 17]byte +var z13931 [1 << 17]byte +var z13932 [1 << 17]byte +var z13933 [1 << 17]byte +var z13934 [1 << 17]byte +var z13935 [1 << 17]byte +var z13936 [1 << 17]byte +var z13937 [1 << 17]byte +var z13938 [1 << 17]byte +var z13939 [1 << 17]byte +var z13940 [1 << 17]byte +var z13941 [1 << 17]byte +var z13942 [1 << 17]byte +var z13943 [1 << 17]byte +var z13944 [1 << 17]byte +var z13945 [1 << 17]byte +var z13946 [1 << 17]byte +var z13947 [1 << 17]byte +var z13948 [1 << 17]byte +var z13949 [1 << 17]byte +var z13950 [1 << 17]byte +var z13951 [1 << 17]byte +var z13952 [1 << 17]byte +var z13953 [1 << 17]byte +var z13954 [1 << 17]byte +var z13955 [1 << 17]byte +var z13956 [1 << 17]byte +var z13957 [1 << 17]byte +var z13958 [1 << 17]byte +var z13959 [1 << 17]byte +var z13960 [1 << 17]byte +var z13961 [1 << 17]byte +var z13962 [1 << 17]byte +var z13963 [1 << 17]byte +var z13964 [1 << 17]byte +var z13965 [1 << 17]byte +var z13966 [1 << 17]byte +var z13967 [1 << 17]byte +var z13968 [1 << 17]byte +var z13969 [1 << 17]byte +var z13970 [1 << 17]byte +var z13971 [1 << 17]byte +var z13972 [1 << 17]byte +var z13973 [1 << 17]byte +var z13974 [1 << 17]byte +var z13975 [1 << 17]byte +var z13976 [1 << 17]byte +var z13977 [1 << 17]byte +var z13978 [1 << 17]byte +var z13979 [1 << 17]byte +var z13980 [1 << 17]byte +var z13981 [1 << 17]byte +var z13982 [1 << 17]byte +var z13983 [1 << 17]byte +var z13984 [1 << 17]byte +var z13985 [1 << 17]byte +var z13986 [1 << 17]byte +var z13987 [1 << 17]byte +var z13988 [1 << 17]byte +var z13989 [1 << 17]byte +var z13990 [1 << 17]byte +var z13991 [1 << 17]byte +var z13992 [1 << 17]byte +var z13993 [1 << 17]byte +var z13994 [1 << 17]byte +var z13995 [1 << 17]byte +var z13996 [1 << 17]byte +var z13997 [1 << 17]byte +var z13998 [1 << 17]byte +var z13999 [1 << 17]byte +var z14000 [1 << 17]byte +var z14001 [1 << 17]byte +var z14002 [1 << 17]byte +var z14003 [1 << 17]byte +var z14004 [1 << 17]byte +var z14005 [1 << 17]byte +var z14006 [1 << 17]byte +var z14007 [1 << 17]byte +var z14008 [1 << 17]byte +var z14009 [1 << 17]byte +var z14010 [1 << 17]byte +var z14011 [1 << 17]byte +var z14012 [1 << 17]byte +var z14013 [1 << 17]byte +var z14014 [1 << 17]byte +var z14015 [1 << 17]byte +var z14016 [1 << 17]byte +var z14017 [1 << 17]byte +var z14018 [1 << 17]byte +var z14019 [1 << 17]byte +var z14020 [1 << 17]byte +var z14021 [1 << 17]byte +var z14022 [1 << 17]byte +var z14023 [1 << 17]byte +var z14024 [1 << 17]byte +var z14025 [1 << 17]byte +var z14026 [1 << 17]byte +var z14027 [1 << 17]byte +var z14028 [1 << 17]byte +var z14029 [1 << 17]byte +var z14030 [1 << 17]byte +var z14031 [1 << 17]byte +var z14032 [1 << 17]byte +var z14033 [1 << 17]byte +var z14034 [1 << 17]byte +var z14035 [1 << 17]byte +var z14036 [1 << 17]byte +var z14037 [1 << 17]byte +var z14038 [1 << 17]byte +var z14039 [1 << 17]byte +var z14040 [1 << 17]byte +var z14041 [1 << 17]byte +var z14042 [1 << 17]byte +var z14043 [1 << 17]byte +var z14044 [1 << 17]byte +var z14045 [1 << 17]byte +var z14046 [1 << 17]byte +var z14047 [1 << 17]byte +var z14048 [1 << 17]byte +var z14049 [1 << 17]byte +var z14050 [1 << 17]byte +var z14051 [1 << 17]byte +var z14052 [1 << 17]byte +var z14053 [1 << 17]byte +var z14054 [1 << 17]byte +var z14055 [1 << 17]byte +var z14056 [1 << 17]byte +var z14057 [1 << 17]byte +var z14058 [1 << 17]byte +var z14059 [1 << 17]byte +var z14060 [1 << 17]byte +var z14061 [1 << 17]byte +var z14062 [1 << 17]byte +var z14063 [1 << 17]byte +var z14064 [1 << 17]byte +var z14065 [1 << 17]byte +var z14066 [1 << 17]byte +var z14067 [1 << 17]byte +var z14068 [1 << 17]byte +var z14069 [1 << 17]byte +var z14070 [1 << 17]byte +var z14071 [1 << 17]byte +var z14072 [1 << 17]byte +var z14073 [1 << 17]byte +var z14074 [1 << 17]byte +var z14075 [1 << 17]byte +var z14076 [1 << 17]byte +var z14077 [1 << 17]byte +var z14078 [1 << 17]byte +var z14079 [1 << 17]byte +var z14080 [1 << 17]byte +var z14081 [1 << 17]byte +var z14082 [1 << 17]byte +var z14083 [1 << 17]byte +var z14084 [1 << 17]byte +var z14085 [1 << 17]byte +var z14086 [1 << 17]byte +var z14087 [1 << 17]byte +var z14088 [1 << 17]byte +var z14089 [1 << 17]byte +var z14090 [1 << 17]byte +var z14091 [1 << 17]byte +var z14092 [1 << 17]byte +var z14093 [1 << 17]byte +var z14094 [1 << 17]byte +var z14095 [1 << 17]byte +var z14096 [1 << 17]byte +var z14097 [1 << 17]byte +var z14098 [1 << 17]byte +var z14099 [1 << 17]byte +var z14100 [1 << 17]byte +var z14101 [1 << 17]byte +var z14102 [1 << 17]byte +var z14103 [1 << 17]byte +var z14104 [1 << 17]byte +var z14105 [1 << 17]byte +var z14106 [1 << 17]byte +var z14107 [1 << 17]byte +var z14108 [1 << 17]byte +var z14109 [1 << 17]byte +var z14110 [1 << 17]byte +var z14111 [1 << 17]byte +var z14112 [1 << 17]byte +var z14113 [1 << 17]byte +var z14114 [1 << 17]byte +var z14115 [1 << 17]byte +var z14116 [1 << 17]byte +var z14117 [1 << 17]byte +var z14118 [1 << 17]byte +var z14119 [1 << 17]byte +var z14120 [1 << 17]byte +var z14121 [1 << 17]byte +var z14122 [1 << 17]byte +var z14123 [1 << 17]byte +var z14124 [1 << 17]byte +var z14125 [1 << 17]byte +var z14126 [1 << 17]byte +var z14127 [1 << 17]byte +var z14128 [1 << 17]byte +var z14129 [1 << 17]byte +var z14130 [1 << 17]byte +var z14131 [1 << 17]byte +var z14132 [1 << 17]byte +var z14133 [1 << 17]byte +var z14134 [1 << 17]byte +var z14135 [1 << 17]byte +var z14136 [1 << 17]byte +var z14137 [1 << 17]byte +var z14138 [1 << 17]byte +var z14139 [1 << 17]byte +var z14140 [1 << 17]byte +var z14141 [1 << 17]byte +var z14142 [1 << 17]byte +var z14143 [1 << 17]byte +var z14144 [1 << 17]byte +var z14145 [1 << 17]byte +var z14146 [1 << 17]byte +var z14147 [1 << 17]byte +var z14148 [1 << 17]byte +var z14149 [1 << 17]byte +var z14150 [1 << 17]byte +var z14151 [1 << 17]byte +var z14152 [1 << 17]byte +var z14153 [1 << 17]byte +var z14154 [1 << 17]byte +var z14155 [1 << 17]byte +var z14156 [1 << 17]byte +var z14157 [1 << 17]byte +var z14158 [1 << 17]byte +var z14159 [1 << 17]byte +var z14160 [1 << 17]byte +var z14161 [1 << 17]byte +var z14162 [1 << 17]byte +var z14163 [1 << 17]byte +var z14164 [1 << 17]byte +var z14165 [1 << 17]byte +var z14166 [1 << 17]byte +var z14167 [1 << 17]byte +var z14168 [1 << 17]byte +var z14169 [1 << 17]byte +var z14170 [1 << 17]byte +var z14171 [1 << 17]byte +var z14172 [1 << 17]byte +var z14173 [1 << 17]byte +var z14174 [1 << 17]byte +var z14175 [1 << 17]byte +var z14176 [1 << 17]byte +var z14177 [1 << 17]byte +var z14178 [1 << 17]byte +var z14179 [1 << 17]byte +var z14180 [1 << 17]byte +var z14181 [1 << 17]byte +var z14182 [1 << 17]byte +var z14183 [1 << 17]byte +var z14184 [1 << 17]byte +var z14185 [1 << 17]byte +var z14186 [1 << 17]byte +var z14187 [1 << 17]byte +var z14188 [1 << 17]byte +var z14189 [1 << 17]byte +var z14190 [1 << 17]byte +var z14191 [1 << 17]byte +var z14192 [1 << 17]byte +var z14193 [1 << 17]byte +var z14194 [1 << 17]byte +var z14195 [1 << 17]byte +var z14196 [1 << 17]byte +var z14197 [1 << 17]byte +var z14198 [1 << 17]byte +var z14199 [1 << 17]byte +var z14200 [1 << 17]byte +var z14201 [1 << 17]byte +var z14202 [1 << 17]byte +var z14203 [1 << 17]byte +var z14204 [1 << 17]byte +var z14205 [1 << 17]byte +var z14206 [1 << 17]byte +var z14207 [1 << 17]byte +var z14208 [1 << 17]byte +var z14209 [1 << 17]byte +var z14210 [1 << 17]byte +var z14211 [1 << 17]byte +var z14212 [1 << 17]byte +var z14213 [1 << 17]byte +var z14214 [1 << 17]byte +var z14215 [1 << 17]byte +var z14216 [1 << 17]byte +var z14217 [1 << 17]byte +var z14218 [1 << 17]byte +var z14219 [1 << 17]byte +var z14220 [1 << 17]byte +var z14221 [1 << 17]byte +var z14222 [1 << 17]byte +var z14223 [1 << 17]byte +var z14224 [1 << 17]byte +var z14225 [1 << 17]byte +var z14226 [1 << 17]byte +var z14227 [1 << 17]byte +var z14228 [1 << 17]byte +var z14229 [1 << 17]byte +var z14230 [1 << 17]byte +var z14231 [1 << 17]byte +var z14232 [1 << 17]byte +var z14233 [1 << 17]byte +var z14234 [1 << 17]byte +var z14235 [1 << 17]byte +var z14236 [1 << 17]byte +var z14237 [1 << 17]byte +var z14238 [1 << 17]byte +var z14239 [1 << 17]byte +var z14240 [1 << 17]byte +var z14241 [1 << 17]byte +var z14242 [1 << 17]byte +var z14243 [1 << 17]byte +var z14244 [1 << 17]byte +var z14245 [1 << 17]byte +var z14246 [1 << 17]byte +var z14247 [1 << 17]byte +var z14248 [1 << 17]byte +var z14249 [1 << 17]byte +var z14250 [1 << 17]byte +var z14251 [1 << 17]byte +var z14252 [1 << 17]byte +var z14253 [1 << 17]byte +var z14254 [1 << 17]byte +var z14255 [1 << 17]byte +var z14256 [1 << 17]byte +var z14257 [1 << 17]byte +var z14258 [1 << 17]byte +var z14259 [1 << 17]byte +var z14260 [1 << 17]byte +var z14261 [1 << 17]byte +var z14262 [1 << 17]byte +var z14263 [1 << 17]byte +var z14264 [1 << 17]byte +var z14265 [1 << 17]byte +var z14266 [1 << 17]byte +var z14267 [1 << 17]byte +var z14268 [1 << 17]byte +var z14269 [1 << 17]byte +var z14270 [1 << 17]byte +var z14271 [1 << 17]byte +var z14272 [1 << 17]byte +var z14273 [1 << 17]byte +var z14274 [1 << 17]byte +var z14275 [1 << 17]byte +var z14276 [1 << 17]byte +var z14277 [1 << 17]byte +var z14278 [1 << 17]byte +var z14279 [1 << 17]byte +var z14280 [1 << 17]byte +var z14281 [1 << 17]byte +var z14282 [1 << 17]byte +var z14283 [1 << 17]byte +var z14284 [1 << 17]byte +var z14285 [1 << 17]byte +var z14286 [1 << 17]byte +var z14287 [1 << 17]byte +var z14288 [1 << 17]byte +var z14289 [1 << 17]byte +var z14290 [1 << 17]byte +var z14291 [1 << 17]byte +var z14292 [1 << 17]byte +var z14293 [1 << 17]byte +var z14294 [1 << 17]byte +var z14295 [1 << 17]byte +var z14296 [1 << 17]byte +var z14297 [1 << 17]byte +var z14298 [1 << 17]byte +var z14299 [1 << 17]byte +var z14300 [1 << 17]byte +var z14301 [1 << 17]byte +var z14302 [1 << 17]byte +var z14303 [1 << 17]byte +var z14304 [1 << 17]byte +var z14305 [1 << 17]byte +var z14306 [1 << 17]byte +var z14307 [1 << 17]byte +var z14308 [1 << 17]byte +var z14309 [1 << 17]byte +var z14310 [1 << 17]byte +var z14311 [1 << 17]byte +var z14312 [1 << 17]byte +var z14313 [1 << 17]byte +var z14314 [1 << 17]byte +var z14315 [1 << 17]byte +var z14316 [1 << 17]byte +var z14317 [1 << 17]byte +var z14318 [1 << 17]byte +var z14319 [1 << 17]byte +var z14320 [1 << 17]byte +var z14321 [1 << 17]byte +var z14322 [1 << 17]byte +var z14323 [1 << 17]byte +var z14324 [1 << 17]byte +var z14325 [1 << 17]byte +var z14326 [1 << 17]byte +var z14327 [1 << 17]byte +var z14328 [1 << 17]byte +var z14329 [1 << 17]byte +var z14330 [1 << 17]byte +var z14331 [1 << 17]byte +var z14332 [1 << 17]byte +var z14333 [1 << 17]byte +var z14334 [1 << 17]byte +var z14335 [1 << 17]byte +var z14336 [1 << 17]byte +var z14337 [1 << 17]byte +var z14338 [1 << 17]byte +var z14339 [1 << 17]byte +var z14340 [1 << 17]byte +var z14341 [1 << 17]byte +var z14342 [1 << 17]byte +var z14343 [1 << 17]byte +var z14344 [1 << 17]byte +var z14345 [1 << 17]byte +var z14346 [1 << 17]byte +var z14347 [1 << 17]byte +var z14348 [1 << 17]byte +var z14349 [1 << 17]byte +var z14350 [1 << 17]byte +var z14351 [1 << 17]byte +var z14352 [1 << 17]byte +var z14353 [1 << 17]byte +var z14354 [1 << 17]byte +var z14355 [1 << 17]byte +var z14356 [1 << 17]byte +var z14357 [1 << 17]byte +var z14358 [1 << 17]byte +var z14359 [1 << 17]byte +var z14360 [1 << 17]byte +var z14361 [1 << 17]byte +var z14362 [1 << 17]byte +var z14363 [1 << 17]byte +var z14364 [1 << 17]byte +var z14365 [1 << 17]byte +var z14366 [1 << 17]byte +var z14367 [1 << 17]byte +var z14368 [1 << 17]byte +var z14369 [1 << 17]byte +var z14370 [1 << 17]byte +var z14371 [1 << 17]byte +var z14372 [1 << 17]byte +var z14373 [1 << 17]byte +var z14374 [1 << 17]byte +var z14375 [1 << 17]byte +var z14376 [1 << 17]byte +var z14377 [1 << 17]byte +var z14378 [1 << 17]byte +var z14379 [1 << 17]byte +var z14380 [1 << 17]byte +var z14381 [1 << 17]byte +var z14382 [1 << 17]byte +var z14383 [1 << 17]byte +var z14384 [1 << 17]byte +var z14385 [1 << 17]byte +var z14386 [1 << 17]byte +var z14387 [1 << 17]byte +var z14388 [1 << 17]byte +var z14389 [1 << 17]byte +var z14390 [1 << 17]byte +var z14391 [1 << 17]byte +var z14392 [1 << 17]byte +var z14393 [1 << 17]byte +var z14394 [1 << 17]byte +var z14395 [1 << 17]byte +var z14396 [1 << 17]byte +var z14397 [1 << 17]byte +var z14398 [1 << 17]byte +var z14399 [1 << 17]byte +var z14400 [1 << 17]byte +var z14401 [1 << 17]byte +var z14402 [1 << 17]byte +var z14403 [1 << 17]byte +var z14404 [1 << 17]byte +var z14405 [1 << 17]byte +var z14406 [1 << 17]byte +var z14407 [1 << 17]byte +var z14408 [1 << 17]byte +var z14409 [1 << 17]byte +var z14410 [1 << 17]byte +var z14411 [1 << 17]byte +var z14412 [1 << 17]byte +var z14413 [1 << 17]byte +var z14414 [1 << 17]byte +var z14415 [1 << 17]byte +var z14416 [1 << 17]byte +var z14417 [1 << 17]byte +var z14418 [1 << 17]byte +var z14419 [1 << 17]byte +var z14420 [1 << 17]byte +var z14421 [1 << 17]byte +var z14422 [1 << 17]byte +var z14423 [1 << 17]byte +var z14424 [1 << 17]byte +var z14425 [1 << 17]byte +var z14426 [1 << 17]byte +var z14427 [1 << 17]byte +var z14428 [1 << 17]byte +var z14429 [1 << 17]byte +var z14430 [1 << 17]byte +var z14431 [1 << 17]byte +var z14432 [1 << 17]byte +var z14433 [1 << 17]byte +var z14434 [1 << 17]byte +var z14435 [1 << 17]byte +var z14436 [1 << 17]byte +var z14437 [1 << 17]byte +var z14438 [1 << 17]byte +var z14439 [1 << 17]byte +var z14440 [1 << 17]byte +var z14441 [1 << 17]byte +var z14442 [1 << 17]byte +var z14443 [1 << 17]byte +var z14444 [1 << 17]byte +var z14445 [1 << 17]byte +var z14446 [1 << 17]byte +var z14447 [1 << 17]byte +var z14448 [1 << 17]byte +var z14449 [1 << 17]byte +var z14450 [1 << 17]byte +var z14451 [1 << 17]byte +var z14452 [1 << 17]byte +var z14453 [1 << 17]byte +var z14454 [1 << 17]byte +var z14455 [1 << 17]byte +var z14456 [1 << 17]byte +var z14457 [1 << 17]byte +var z14458 [1 << 17]byte +var z14459 [1 << 17]byte +var z14460 [1 << 17]byte +var z14461 [1 << 17]byte +var z14462 [1 << 17]byte +var z14463 [1 << 17]byte +var z14464 [1 << 17]byte +var z14465 [1 << 17]byte +var z14466 [1 << 17]byte +var z14467 [1 << 17]byte +var z14468 [1 << 17]byte +var z14469 [1 << 17]byte +var z14470 [1 << 17]byte +var z14471 [1 << 17]byte +var z14472 [1 << 17]byte +var z14473 [1 << 17]byte +var z14474 [1 << 17]byte +var z14475 [1 << 17]byte +var z14476 [1 << 17]byte +var z14477 [1 << 17]byte +var z14478 [1 << 17]byte +var z14479 [1 << 17]byte +var z14480 [1 << 17]byte +var z14481 [1 << 17]byte +var z14482 [1 << 17]byte +var z14483 [1 << 17]byte +var z14484 [1 << 17]byte +var z14485 [1 << 17]byte +var z14486 [1 << 17]byte +var z14487 [1 << 17]byte +var z14488 [1 << 17]byte +var z14489 [1 << 17]byte +var z14490 [1 << 17]byte +var z14491 [1 << 17]byte +var z14492 [1 << 17]byte +var z14493 [1 << 17]byte +var z14494 [1 << 17]byte +var z14495 [1 << 17]byte +var z14496 [1 << 17]byte +var z14497 [1 << 17]byte +var z14498 [1 << 17]byte +var z14499 [1 << 17]byte +var z14500 [1 << 17]byte +var z14501 [1 << 17]byte +var z14502 [1 << 17]byte +var z14503 [1 << 17]byte +var z14504 [1 << 17]byte +var z14505 [1 << 17]byte +var z14506 [1 << 17]byte +var z14507 [1 << 17]byte +var z14508 [1 << 17]byte +var z14509 [1 << 17]byte +var z14510 [1 << 17]byte +var z14511 [1 << 17]byte +var z14512 [1 << 17]byte +var z14513 [1 << 17]byte +var z14514 [1 << 17]byte +var z14515 [1 << 17]byte +var z14516 [1 << 17]byte +var z14517 [1 << 17]byte +var z14518 [1 << 17]byte +var z14519 [1 << 17]byte +var z14520 [1 << 17]byte +var z14521 [1 << 17]byte +var z14522 [1 << 17]byte +var z14523 [1 << 17]byte +var z14524 [1 << 17]byte +var z14525 [1 << 17]byte +var z14526 [1 << 17]byte +var z14527 [1 << 17]byte +var z14528 [1 << 17]byte +var z14529 [1 << 17]byte +var z14530 [1 << 17]byte +var z14531 [1 << 17]byte +var z14532 [1 << 17]byte +var z14533 [1 << 17]byte +var z14534 [1 << 17]byte +var z14535 [1 << 17]byte +var z14536 [1 << 17]byte +var z14537 [1 << 17]byte +var z14538 [1 << 17]byte +var z14539 [1 << 17]byte +var z14540 [1 << 17]byte +var z14541 [1 << 17]byte +var z14542 [1 << 17]byte +var z14543 [1 << 17]byte +var z14544 [1 << 17]byte +var z14545 [1 << 17]byte +var z14546 [1 << 17]byte +var z14547 [1 << 17]byte +var z14548 [1 << 17]byte +var z14549 [1 << 17]byte +var z14550 [1 << 17]byte +var z14551 [1 << 17]byte +var z14552 [1 << 17]byte +var z14553 [1 << 17]byte +var z14554 [1 << 17]byte +var z14555 [1 << 17]byte +var z14556 [1 << 17]byte +var z14557 [1 << 17]byte +var z14558 [1 << 17]byte +var z14559 [1 << 17]byte +var z14560 [1 << 17]byte +var z14561 [1 << 17]byte +var z14562 [1 << 17]byte +var z14563 [1 << 17]byte +var z14564 [1 << 17]byte +var z14565 [1 << 17]byte +var z14566 [1 << 17]byte +var z14567 [1 << 17]byte +var z14568 [1 << 17]byte +var z14569 [1 << 17]byte +var z14570 [1 << 17]byte +var z14571 [1 << 17]byte +var z14572 [1 << 17]byte +var z14573 [1 << 17]byte +var z14574 [1 << 17]byte +var z14575 [1 << 17]byte +var z14576 [1 << 17]byte +var z14577 [1 << 17]byte +var z14578 [1 << 17]byte +var z14579 [1 << 17]byte +var z14580 [1 << 17]byte +var z14581 [1 << 17]byte +var z14582 [1 << 17]byte +var z14583 [1 << 17]byte +var z14584 [1 << 17]byte +var z14585 [1 << 17]byte +var z14586 [1 << 17]byte +var z14587 [1 << 17]byte +var z14588 [1 << 17]byte +var z14589 [1 << 17]byte +var z14590 [1 << 17]byte +var z14591 [1 << 17]byte +var z14592 [1 << 17]byte +var z14593 [1 << 17]byte +var z14594 [1 << 17]byte +var z14595 [1 << 17]byte +var z14596 [1 << 17]byte +var z14597 [1 << 17]byte +var z14598 [1 << 17]byte +var z14599 [1 << 17]byte +var z14600 [1 << 17]byte +var z14601 [1 << 17]byte +var z14602 [1 << 17]byte +var z14603 [1 << 17]byte +var z14604 [1 << 17]byte +var z14605 [1 << 17]byte +var z14606 [1 << 17]byte +var z14607 [1 << 17]byte +var z14608 [1 << 17]byte +var z14609 [1 << 17]byte +var z14610 [1 << 17]byte +var z14611 [1 << 17]byte +var z14612 [1 << 17]byte +var z14613 [1 << 17]byte +var z14614 [1 << 17]byte +var z14615 [1 << 17]byte +var z14616 [1 << 17]byte +var z14617 [1 << 17]byte +var z14618 [1 << 17]byte +var z14619 [1 << 17]byte +var z14620 [1 << 17]byte +var z14621 [1 << 17]byte +var z14622 [1 << 17]byte +var z14623 [1 << 17]byte +var z14624 [1 << 17]byte +var z14625 [1 << 17]byte +var z14626 [1 << 17]byte +var z14627 [1 << 17]byte +var z14628 [1 << 17]byte +var z14629 [1 << 17]byte +var z14630 [1 << 17]byte +var z14631 [1 << 17]byte +var z14632 [1 << 17]byte +var z14633 [1 << 17]byte +var z14634 [1 << 17]byte +var z14635 [1 << 17]byte +var z14636 [1 << 17]byte +var z14637 [1 << 17]byte +var z14638 [1 << 17]byte +var z14639 [1 << 17]byte +var z14640 [1 << 17]byte +var z14641 [1 << 17]byte +var z14642 [1 << 17]byte +var z14643 [1 << 17]byte +var z14644 [1 << 17]byte +var z14645 [1 << 17]byte +var z14646 [1 << 17]byte +var z14647 [1 << 17]byte +var z14648 [1 << 17]byte +var z14649 [1 << 17]byte +var z14650 [1 << 17]byte +var z14651 [1 << 17]byte +var z14652 [1 << 17]byte +var z14653 [1 << 17]byte +var z14654 [1 << 17]byte +var z14655 [1 << 17]byte +var z14656 [1 << 17]byte +var z14657 [1 << 17]byte +var z14658 [1 << 17]byte +var z14659 [1 << 17]byte +var z14660 [1 << 17]byte +var z14661 [1 << 17]byte +var z14662 [1 << 17]byte +var z14663 [1 << 17]byte +var z14664 [1 << 17]byte +var z14665 [1 << 17]byte +var z14666 [1 << 17]byte +var z14667 [1 << 17]byte +var z14668 [1 << 17]byte +var z14669 [1 << 17]byte +var z14670 [1 << 17]byte +var z14671 [1 << 17]byte +var z14672 [1 << 17]byte +var z14673 [1 << 17]byte +var z14674 [1 << 17]byte +var z14675 [1 << 17]byte +var z14676 [1 << 17]byte +var z14677 [1 << 17]byte +var z14678 [1 << 17]byte +var z14679 [1 << 17]byte +var z14680 [1 << 17]byte +var z14681 [1 << 17]byte +var z14682 [1 << 17]byte +var z14683 [1 << 17]byte +var z14684 [1 << 17]byte +var z14685 [1 << 17]byte +var z14686 [1 << 17]byte +var z14687 [1 << 17]byte +var z14688 [1 << 17]byte +var z14689 [1 << 17]byte +var z14690 [1 << 17]byte +var z14691 [1 << 17]byte +var z14692 [1 << 17]byte +var z14693 [1 << 17]byte +var z14694 [1 << 17]byte +var z14695 [1 << 17]byte +var z14696 [1 << 17]byte +var z14697 [1 << 17]byte +var z14698 [1 << 17]byte +var z14699 [1 << 17]byte +var z14700 [1 << 17]byte +var z14701 [1 << 17]byte +var z14702 [1 << 17]byte +var z14703 [1 << 17]byte +var z14704 [1 << 17]byte +var z14705 [1 << 17]byte +var z14706 [1 << 17]byte +var z14707 [1 << 17]byte +var z14708 [1 << 17]byte +var z14709 [1 << 17]byte +var z14710 [1 << 17]byte +var z14711 [1 << 17]byte +var z14712 [1 << 17]byte +var z14713 [1 << 17]byte +var z14714 [1 << 17]byte +var z14715 [1 << 17]byte +var z14716 [1 << 17]byte +var z14717 [1 << 17]byte +var z14718 [1 << 17]byte +var z14719 [1 << 17]byte +var z14720 [1 << 17]byte +var z14721 [1 << 17]byte +var z14722 [1 << 17]byte +var z14723 [1 << 17]byte +var z14724 [1 << 17]byte +var z14725 [1 << 17]byte +var z14726 [1 << 17]byte +var z14727 [1 << 17]byte +var z14728 [1 << 17]byte +var z14729 [1 << 17]byte +var z14730 [1 << 17]byte +var z14731 [1 << 17]byte +var z14732 [1 << 17]byte +var z14733 [1 << 17]byte +var z14734 [1 << 17]byte +var z14735 [1 << 17]byte +var z14736 [1 << 17]byte +var z14737 [1 << 17]byte +var z14738 [1 << 17]byte +var z14739 [1 << 17]byte +var z14740 [1 << 17]byte +var z14741 [1 << 17]byte +var z14742 [1 << 17]byte +var z14743 [1 << 17]byte +var z14744 [1 << 17]byte +var z14745 [1 << 17]byte +var z14746 [1 << 17]byte +var z14747 [1 << 17]byte +var z14748 [1 << 17]byte +var z14749 [1 << 17]byte +var z14750 [1 << 17]byte +var z14751 [1 << 17]byte +var z14752 [1 << 17]byte +var z14753 [1 << 17]byte +var z14754 [1 << 17]byte +var z14755 [1 << 17]byte +var z14756 [1 << 17]byte +var z14757 [1 << 17]byte +var z14758 [1 << 17]byte +var z14759 [1 << 17]byte +var z14760 [1 << 17]byte +var z14761 [1 << 17]byte +var z14762 [1 << 17]byte +var z14763 [1 << 17]byte +var z14764 [1 << 17]byte +var z14765 [1 << 17]byte +var z14766 [1 << 17]byte +var z14767 [1 << 17]byte +var z14768 [1 << 17]byte +var z14769 [1 << 17]byte +var z14770 [1 << 17]byte +var z14771 [1 << 17]byte +var z14772 [1 << 17]byte +var z14773 [1 << 17]byte +var z14774 [1 << 17]byte +var z14775 [1 << 17]byte +var z14776 [1 << 17]byte +var z14777 [1 << 17]byte +var z14778 [1 << 17]byte +var z14779 [1 << 17]byte +var z14780 [1 << 17]byte +var z14781 [1 << 17]byte +var z14782 [1 << 17]byte +var z14783 [1 << 17]byte +var z14784 [1 << 17]byte +var z14785 [1 << 17]byte +var z14786 [1 << 17]byte +var z14787 [1 << 17]byte +var z14788 [1 << 17]byte +var z14789 [1 << 17]byte +var z14790 [1 << 17]byte +var z14791 [1 << 17]byte +var z14792 [1 << 17]byte +var z14793 [1 << 17]byte +var z14794 [1 << 17]byte +var z14795 [1 << 17]byte +var z14796 [1 << 17]byte +var z14797 [1 << 17]byte +var z14798 [1 << 17]byte +var z14799 [1 << 17]byte +var z14800 [1 << 17]byte +var z14801 [1 << 17]byte +var z14802 [1 << 17]byte +var z14803 [1 << 17]byte +var z14804 [1 << 17]byte +var z14805 [1 << 17]byte +var z14806 [1 << 17]byte +var z14807 [1 << 17]byte +var z14808 [1 << 17]byte +var z14809 [1 << 17]byte +var z14810 [1 << 17]byte +var z14811 [1 << 17]byte +var z14812 [1 << 17]byte +var z14813 [1 << 17]byte +var z14814 [1 << 17]byte +var z14815 [1 << 17]byte +var z14816 [1 << 17]byte +var z14817 [1 << 17]byte +var z14818 [1 << 17]byte +var z14819 [1 << 17]byte +var z14820 [1 << 17]byte +var z14821 [1 << 17]byte +var z14822 [1 << 17]byte +var z14823 [1 << 17]byte +var z14824 [1 << 17]byte +var z14825 [1 << 17]byte +var z14826 [1 << 17]byte +var z14827 [1 << 17]byte +var z14828 [1 << 17]byte +var z14829 [1 << 17]byte +var z14830 [1 << 17]byte +var z14831 [1 << 17]byte +var z14832 [1 << 17]byte +var z14833 [1 << 17]byte +var z14834 [1 << 17]byte +var z14835 [1 << 17]byte +var z14836 [1 << 17]byte +var z14837 [1 << 17]byte +var z14838 [1 << 17]byte +var z14839 [1 << 17]byte +var z14840 [1 << 17]byte +var z14841 [1 << 17]byte +var z14842 [1 << 17]byte +var z14843 [1 << 17]byte +var z14844 [1 << 17]byte +var z14845 [1 << 17]byte +var z14846 [1 << 17]byte +var z14847 [1 << 17]byte +var z14848 [1 << 17]byte +var z14849 [1 << 17]byte +var z14850 [1 << 17]byte +var z14851 [1 << 17]byte +var z14852 [1 << 17]byte +var z14853 [1 << 17]byte +var z14854 [1 << 17]byte +var z14855 [1 << 17]byte +var z14856 [1 << 17]byte +var z14857 [1 << 17]byte +var z14858 [1 << 17]byte +var z14859 [1 << 17]byte +var z14860 [1 << 17]byte +var z14861 [1 << 17]byte +var z14862 [1 << 17]byte +var z14863 [1 << 17]byte +var z14864 [1 << 17]byte +var z14865 [1 << 17]byte +var z14866 [1 << 17]byte +var z14867 [1 << 17]byte +var z14868 [1 << 17]byte +var z14869 [1 << 17]byte +var z14870 [1 << 17]byte +var z14871 [1 << 17]byte +var z14872 [1 << 17]byte +var z14873 [1 << 17]byte +var z14874 [1 << 17]byte +var z14875 [1 << 17]byte +var z14876 [1 << 17]byte +var z14877 [1 << 17]byte +var z14878 [1 << 17]byte +var z14879 [1 << 17]byte +var z14880 [1 << 17]byte +var z14881 [1 << 17]byte +var z14882 [1 << 17]byte +var z14883 [1 << 17]byte +var z14884 [1 << 17]byte +var z14885 [1 << 17]byte +var z14886 [1 << 17]byte +var z14887 [1 << 17]byte +var z14888 [1 << 17]byte +var z14889 [1 << 17]byte +var z14890 [1 << 17]byte +var z14891 [1 << 17]byte +var z14892 [1 << 17]byte +var z14893 [1 << 17]byte +var z14894 [1 << 17]byte +var z14895 [1 << 17]byte +var z14896 [1 << 17]byte +var z14897 [1 << 17]byte +var z14898 [1 << 17]byte +var z14899 [1 << 17]byte +var z14900 [1 << 17]byte +var z14901 [1 << 17]byte +var z14902 [1 << 17]byte +var z14903 [1 << 17]byte +var z14904 [1 << 17]byte +var z14905 [1 << 17]byte +var z14906 [1 << 17]byte +var z14907 [1 << 17]byte +var z14908 [1 << 17]byte +var z14909 [1 << 17]byte +var z14910 [1 << 17]byte +var z14911 [1 << 17]byte +var z14912 [1 << 17]byte +var z14913 [1 << 17]byte +var z14914 [1 << 17]byte +var z14915 [1 << 17]byte +var z14916 [1 << 17]byte +var z14917 [1 << 17]byte +var z14918 [1 << 17]byte +var z14919 [1 << 17]byte +var z14920 [1 << 17]byte +var z14921 [1 << 17]byte +var z14922 [1 << 17]byte +var z14923 [1 << 17]byte +var z14924 [1 << 17]byte +var z14925 [1 << 17]byte +var z14926 [1 << 17]byte +var z14927 [1 << 17]byte +var z14928 [1 << 17]byte +var z14929 [1 << 17]byte +var z14930 [1 << 17]byte +var z14931 [1 << 17]byte +var z14932 [1 << 17]byte +var z14933 [1 << 17]byte +var z14934 [1 << 17]byte +var z14935 [1 << 17]byte +var z14936 [1 << 17]byte +var z14937 [1 << 17]byte +var z14938 [1 << 17]byte +var z14939 [1 << 17]byte +var z14940 [1 << 17]byte +var z14941 [1 << 17]byte +var z14942 [1 << 17]byte +var z14943 [1 << 17]byte +var z14944 [1 << 17]byte +var z14945 [1 << 17]byte +var z14946 [1 << 17]byte +var z14947 [1 << 17]byte +var z14948 [1 << 17]byte +var z14949 [1 << 17]byte +var z14950 [1 << 17]byte +var z14951 [1 << 17]byte +var z14952 [1 << 17]byte +var z14953 [1 << 17]byte +var z14954 [1 << 17]byte +var z14955 [1 << 17]byte +var z14956 [1 << 17]byte +var z14957 [1 << 17]byte +var z14958 [1 << 17]byte +var z14959 [1 << 17]byte +var z14960 [1 << 17]byte +var z14961 [1 << 17]byte +var z14962 [1 << 17]byte +var z14963 [1 << 17]byte +var z14964 [1 << 17]byte +var z14965 [1 << 17]byte +var z14966 [1 << 17]byte +var z14967 [1 << 17]byte +var z14968 [1 << 17]byte +var z14969 [1 << 17]byte +var z14970 [1 << 17]byte +var z14971 [1 << 17]byte +var z14972 [1 << 17]byte +var z14973 [1 << 17]byte +var z14974 [1 << 17]byte +var z14975 [1 << 17]byte +var z14976 [1 << 17]byte +var z14977 [1 << 17]byte +var z14978 [1 << 17]byte +var z14979 [1 << 17]byte +var z14980 [1 << 17]byte +var z14981 [1 << 17]byte +var z14982 [1 << 17]byte +var z14983 [1 << 17]byte +var z14984 [1 << 17]byte +var z14985 [1 << 17]byte +var z14986 [1 << 17]byte +var z14987 [1 << 17]byte +var z14988 [1 << 17]byte +var z14989 [1 << 17]byte +var z14990 [1 << 17]byte +var z14991 [1 << 17]byte +var z14992 [1 << 17]byte +var z14993 [1 << 17]byte +var z14994 [1 << 17]byte +var z14995 [1 << 17]byte +var z14996 [1 << 17]byte +var z14997 [1 << 17]byte +var z14998 [1 << 17]byte +var z14999 [1 << 17]byte +var z15000 [1 << 17]byte +var z15001 [1 << 17]byte +var z15002 [1 << 17]byte +var z15003 [1 << 17]byte +var z15004 [1 << 17]byte +var z15005 [1 << 17]byte +var z15006 [1 << 17]byte +var z15007 [1 << 17]byte +var z15008 [1 << 17]byte +var z15009 [1 << 17]byte +var z15010 [1 << 17]byte +var z15011 [1 << 17]byte +var z15012 [1 << 17]byte +var z15013 [1 << 17]byte +var z15014 [1 << 17]byte +var z15015 [1 << 17]byte +var z15016 [1 << 17]byte +var z15017 [1 << 17]byte +var z15018 [1 << 17]byte +var z15019 [1 << 17]byte +var z15020 [1 << 17]byte +var z15021 [1 << 17]byte +var z15022 [1 << 17]byte +var z15023 [1 << 17]byte +var z15024 [1 << 17]byte +var z15025 [1 << 17]byte +var z15026 [1 << 17]byte +var z15027 [1 << 17]byte +var z15028 [1 << 17]byte +var z15029 [1 << 17]byte +var z15030 [1 << 17]byte +var z15031 [1 << 17]byte +var z15032 [1 << 17]byte +var z15033 [1 << 17]byte +var z15034 [1 << 17]byte +var z15035 [1 << 17]byte +var z15036 [1 << 17]byte +var z15037 [1 << 17]byte +var z15038 [1 << 17]byte +var z15039 [1 << 17]byte +var z15040 [1 << 17]byte +var z15041 [1 << 17]byte +var z15042 [1 << 17]byte +var z15043 [1 << 17]byte +var z15044 [1 << 17]byte +var z15045 [1 << 17]byte +var z15046 [1 << 17]byte +var z15047 [1 << 17]byte +var z15048 [1 << 17]byte +var z15049 [1 << 17]byte +var z15050 [1 << 17]byte +var z15051 [1 << 17]byte +var z15052 [1 << 17]byte +var z15053 [1 << 17]byte +var z15054 [1 << 17]byte +var z15055 [1 << 17]byte +var z15056 [1 << 17]byte +var z15057 [1 << 17]byte +var z15058 [1 << 17]byte +var z15059 [1 << 17]byte +var z15060 [1 << 17]byte +var z15061 [1 << 17]byte +var z15062 [1 << 17]byte +var z15063 [1 << 17]byte +var z15064 [1 << 17]byte +var z15065 [1 << 17]byte +var z15066 [1 << 17]byte +var z15067 [1 << 17]byte +var z15068 [1 << 17]byte +var z15069 [1 << 17]byte +var z15070 [1 << 17]byte +var z15071 [1 << 17]byte +var z15072 [1 << 17]byte +var z15073 [1 << 17]byte +var z15074 [1 << 17]byte +var z15075 [1 << 17]byte +var z15076 [1 << 17]byte +var z15077 [1 << 17]byte +var z15078 [1 << 17]byte +var z15079 [1 << 17]byte +var z15080 [1 << 17]byte +var z15081 [1 << 17]byte +var z15082 [1 << 17]byte +var z15083 [1 << 17]byte +var z15084 [1 << 17]byte +var z15085 [1 << 17]byte +var z15086 [1 << 17]byte +var z15087 [1 << 17]byte +var z15088 [1 << 17]byte +var z15089 [1 << 17]byte +var z15090 [1 << 17]byte +var z15091 [1 << 17]byte +var z15092 [1 << 17]byte +var z15093 [1 << 17]byte +var z15094 [1 << 17]byte +var z15095 [1 << 17]byte +var z15096 [1 << 17]byte +var z15097 [1 << 17]byte +var z15098 [1 << 17]byte +var z15099 [1 << 17]byte +var z15100 [1 << 17]byte +var z15101 [1 << 17]byte +var z15102 [1 << 17]byte +var z15103 [1 << 17]byte +var z15104 [1 << 17]byte +var z15105 [1 << 17]byte +var z15106 [1 << 17]byte +var z15107 [1 << 17]byte +var z15108 [1 << 17]byte +var z15109 [1 << 17]byte +var z15110 [1 << 17]byte +var z15111 [1 << 17]byte +var z15112 [1 << 17]byte +var z15113 [1 << 17]byte +var z15114 [1 << 17]byte +var z15115 [1 << 17]byte +var z15116 [1 << 17]byte +var z15117 [1 << 17]byte +var z15118 [1 << 17]byte +var z15119 [1 << 17]byte +var z15120 [1 << 17]byte +var z15121 [1 << 17]byte +var z15122 [1 << 17]byte +var z15123 [1 << 17]byte +var z15124 [1 << 17]byte +var z15125 [1 << 17]byte +var z15126 [1 << 17]byte +var z15127 [1 << 17]byte +var z15128 [1 << 17]byte +var z15129 [1 << 17]byte +var z15130 [1 << 17]byte +var z15131 [1 << 17]byte +var z15132 [1 << 17]byte +var z15133 [1 << 17]byte +var z15134 [1 << 17]byte +var z15135 [1 << 17]byte +var z15136 [1 << 17]byte +var z15137 [1 << 17]byte +var z15138 [1 << 17]byte +var z15139 [1 << 17]byte +var z15140 [1 << 17]byte +var z15141 [1 << 17]byte +var z15142 [1 << 17]byte +var z15143 [1 << 17]byte +var z15144 [1 << 17]byte +var z15145 [1 << 17]byte +var z15146 [1 << 17]byte +var z15147 [1 << 17]byte +var z15148 [1 << 17]byte +var z15149 [1 << 17]byte +var z15150 [1 << 17]byte +var z15151 [1 << 17]byte +var z15152 [1 << 17]byte +var z15153 [1 << 17]byte +var z15154 [1 << 17]byte +var z15155 [1 << 17]byte +var z15156 [1 << 17]byte +var z15157 [1 << 17]byte +var z15158 [1 << 17]byte +var z15159 [1 << 17]byte +var z15160 [1 << 17]byte +var z15161 [1 << 17]byte +var z15162 [1 << 17]byte +var z15163 [1 << 17]byte +var z15164 [1 << 17]byte +var z15165 [1 << 17]byte +var z15166 [1 << 17]byte +var z15167 [1 << 17]byte +var z15168 [1 << 17]byte +var z15169 [1 << 17]byte +var z15170 [1 << 17]byte +var z15171 [1 << 17]byte +var z15172 [1 << 17]byte +var z15173 [1 << 17]byte +var z15174 [1 << 17]byte +var z15175 [1 << 17]byte +var z15176 [1 << 17]byte +var z15177 [1 << 17]byte +var z15178 [1 << 17]byte +var z15179 [1 << 17]byte +var z15180 [1 << 17]byte +var z15181 [1 << 17]byte +var z15182 [1 << 17]byte +var z15183 [1 << 17]byte +var z15184 [1 << 17]byte +var z15185 [1 << 17]byte +var z15186 [1 << 17]byte +var z15187 [1 << 17]byte +var z15188 [1 << 17]byte +var z15189 [1 << 17]byte +var z15190 [1 << 17]byte +var z15191 [1 << 17]byte +var z15192 [1 << 17]byte +var z15193 [1 << 17]byte +var z15194 [1 << 17]byte +var z15195 [1 << 17]byte +var z15196 [1 << 17]byte +var z15197 [1 << 17]byte +var z15198 [1 << 17]byte +var z15199 [1 << 17]byte +var z15200 [1 << 17]byte +var z15201 [1 << 17]byte +var z15202 [1 << 17]byte +var z15203 [1 << 17]byte +var z15204 [1 << 17]byte +var z15205 [1 << 17]byte +var z15206 [1 << 17]byte +var z15207 [1 << 17]byte +var z15208 [1 << 17]byte +var z15209 [1 << 17]byte +var z15210 [1 << 17]byte +var z15211 [1 << 17]byte +var z15212 [1 << 17]byte +var z15213 [1 << 17]byte +var z15214 [1 << 17]byte +var z15215 [1 << 17]byte +var z15216 [1 << 17]byte +var z15217 [1 << 17]byte +var z15218 [1 << 17]byte +var z15219 [1 << 17]byte +var z15220 [1 << 17]byte +var z15221 [1 << 17]byte +var z15222 [1 << 17]byte +var z15223 [1 << 17]byte +var z15224 [1 << 17]byte +var z15225 [1 << 17]byte +var z15226 [1 << 17]byte +var z15227 [1 << 17]byte +var z15228 [1 << 17]byte +var z15229 [1 << 17]byte +var z15230 [1 << 17]byte +var z15231 [1 << 17]byte +var z15232 [1 << 17]byte +var z15233 [1 << 17]byte +var z15234 [1 << 17]byte +var z15235 [1 << 17]byte +var z15236 [1 << 17]byte +var z15237 [1 << 17]byte +var z15238 [1 << 17]byte +var z15239 [1 << 17]byte +var z15240 [1 << 17]byte +var z15241 [1 << 17]byte +var z15242 [1 << 17]byte +var z15243 [1 << 17]byte +var z15244 [1 << 17]byte +var z15245 [1 << 17]byte +var z15246 [1 << 17]byte +var z15247 [1 << 17]byte +var z15248 [1 << 17]byte +var z15249 [1 << 17]byte +var z15250 [1 << 17]byte +var z15251 [1 << 17]byte +var z15252 [1 << 17]byte +var z15253 [1 << 17]byte +var z15254 [1 << 17]byte +var z15255 [1 << 17]byte +var z15256 [1 << 17]byte +var z15257 [1 << 17]byte +var z15258 [1 << 17]byte +var z15259 [1 << 17]byte +var z15260 [1 << 17]byte +var z15261 [1 << 17]byte +var z15262 [1 << 17]byte +var z15263 [1 << 17]byte +var z15264 [1 << 17]byte +var z15265 [1 << 17]byte +var z15266 [1 << 17]byte +var z15267 [1 << 17]byte +var z15268 [1 << 17]byte +var z15269 [1 << 17]byte +var z15270 [1 << 17]byte +var z15271 [1 << 17]byte +var z15272 [1 << 17]byte +var z15273 [1 << 17]byte +var z15274 [1 << 17]byte +var z15275 [1 << 17]byte +var z15276 [1 << 17]byte +var z15277 [1 << 17]byte +var z15278 [1 << 17]byte +var z15279 [1 << 17]byte +var z15280 [1 << 17]byte +var z15281 [1 << 17]byte +var z15282 [1 << 17]byte +var z15283 [1 << 17]byte +var z15284 [1 << 17]byte +var z15285 [1 << 17]byte +var z15286 [1 << 17]byte +var z15287 [1 << 17]byte +var z15288 [1 << 17]byte +var z15289 [1 << 17]byte +var z15290 [1 << 17]byte +var z15291 [1 << 17]byte +var z15292 [1 << 17]byte +var z15293 [1 << 17]byte +var z15294 [1 << 17]byte +var z15295 [1 << 17]byte +var z15296 [1 << 17]byte +var z15297 [1 << 17]byte +var z15298 [1 << 17]byte +var z15299 [1 << 17]byte +var z15300 [1 << 17]byte +var z15301 [1 << 17]byte +var z15302 [1 << 17]byte +var z15303 [1 << 17]byte +var z15304 [1 << 17]byte +var z15305 [1 << 17]byte +var z15306 [1 << 17]byte +var z15307 [1 << 17]byte +var z15308 [1 << 17]byte +var z15309 [1 << 17]byte +var z15310 [1 << 17]byte +var z15311 [1 << 17]byte +var z15312 [1 << 17]byte +var z15313 [1 << 17]byte +var z15314 [1 << 17]byte +var z15315 [1 << 17]byte +var z15316 [1 << 17]byte +var z15317 [1 << 17]byte +var z15318 [1 << 17]byte +var z15319 [1 << 17]byte +var z15320 [1 << 17]byte +var z15321 [1 << 17]byte +var z15322 [1 << 17]byte +var z15323 [1 << 17]byte +var z15324 [1 << 17]byte +var z15325 [1 << 17]byte +var z15326 [1 << 17]byte +var z15327 [1 << 17]byte +var z15328 [1 << 17]byte +var z15329 [1 << 17]byte +var z15330 [1 << 17]byte +var z15331 [1 << 17]byte +var z15332 [1 << 17]byte +var z15333 [1 << 17]byte +var z15334 [1 << 17]byte +var z15335 [1 << 17]byte +var z15336 [1 << 17]byte +var z15337 [1 << 17]byte +var z15338 [1 << 17]byte +var z15339 [1 << 17]byte +var z15340 [1 << 17]byte +var z15341 [1 << 17]byte +var z15342 [1 << 17]byte +var z15343 [1 << 17]byte +var z15344 [1 << 17]byte +var z15345 [1 << 17]byte +var z15346 [1 << 17]byte +var z15347 [1 << 17]byte +var z15348 [1 << 17]byte +var z15349 [1 << 17]byte +var z15350 [1 << 17]byte +var z15351 [1 << 17]byte +var z15352 [1 << 17]byte +var z15353 [1 << 17]byte +var z15354 [1 << 17]byte +var z15355 [1 << 17]byte +var z15356 [1 << 17]byte +var z15357 [1 << 17]byte +var z15358 [1 << 17]byte +var z15359 [1 << 17]byte +var z15360 [1 << 17]byte +var z15361 [1 << 17]byte +var z15362 [1 << 17]byte +var z15363 [1 << 17]byte +var z15364 [1 << 17]byte +var z15365 [1 << 17]byte +var z15366 [1 << 17]byte +var z15367 [1 << 17]byte +var z15368 [1 << 17]byte +var z15369 [1 << 17]byte +var z15370 [1 << 17]byte +var z15371 [1 << 17]byte +var z15372 [1 << 17]byte +var z15373 [1 << 17]byte +var z15374 [1 << 17]byte +var z15375 [1 << 17]byte +var z15376 [1 << 17]byte +var z15377 [1 << 17]byte +var z15378 [1 << 17]byte +var z15379 [1 << 17]byte +var z15380 [1 << 17]byte +var z15381 [1 << 17]byte +var z15382 [1 << 17]byte +var z15383 [1 << 17]byte +var z15384 [1 << 17]byte +var z15385 [1 << 17]byte +var z15386 [1 << 17]byte +var z15387 [1 << 17]byte +var z15388 [1 << 17]byte +var z15389 [1 << 17]byte +var z15390 [1 << 17]byte +var z15391 [1 << 17]byte +var z15392 [1 << 17]byte +var z15393 [1 << 17]byte +var z15394 [1 << 17]byte +var z15395 [1 << 17]byte +var z15396 [1 << 17]byte +var z15397 [1 << 17]byte +var z15398 [1 << 17]byte +var z15399 [1 << 17]byte +var z15400 [1 << 17]byte +var z15401 [1 << 17]byte +var z15402 [1 << 17]byte +var z15403 [1 << 17]byte +var z15404 [1 << 17]byte +var z15405 [1 << 17]byte +var z15406 [1 << 17]byte +var z15407 [1 << 17]byte +var z15408 [1 << 17]byte +var z15409 [1 << 17]byte +var z15410 [1 << 17]byte +var z15411 [1 << 17]byte +var z15412 [1 << 17]byte +var z15413 [1 << 17]byte +var z15414 [1 << 17]byte +var z15415 [1 << 17]byte +var z15416 [1 << 17]byte +var z15417 [1 << 17]byte +var z15418 [1 << 17]byte +var z15419 [1 << 17]byte +var z15420 [1 << 17]byte +var z15421 [1 << 17]byte +var z15422 [1 << 17]byte +var z15423 [1 << 17]byte +var z15424 [1 << 17]byte +var z15425 [1 << 17]byte +var z15426 [1 << 17]byte +var z15427 [1 << 17]byte +var z15428 [1 << 17]byte +var z15429 [1 << 17]byte +var z15430 [1 << 17]byte +var z15431 [1 << 17]byte +var z15432 [1 << 17]byte +var z15433 [1 << 17]byte +var z15434 [1 << 17]byte +var z15435 [1 << 17]byte +var z15436 [1 << 17]byte +var z15437 [1 << 17]byte +var z15438 [1 << 17]byte +var z15439 [1 << 17]byte +var z15440 [1 << 17]byte +var z15441 [1 << 17]byte +var z15442 [1 << 17]byte +var z15443 [1 << 17]byte +var z15444 [1 << 17]byte +var z15445 [1 << 17]byte +var z15446 [1 << 17]byte +var z15447 [1 << 17]byte +var z15448 [1 << 17]byte +var z15449 [1 << 17]byte +var z15450 [1 << 17]byte +var z15451 [1 << 17]byte +var z15452 [1 << 17]byte +var z15453 [1 << 17]byte +var z15454 [1 << 17]byte +var z15455 [1 << 17]byte +var z15456 [1 << 17]byte +var z15457 [1 << 17]byte +var z15458 [1 << 17]byte +var z15459 [1 << 17]byte +var z15460 [1 << 17]byte +var z15461 [1 << 17]byte +var z15462 [1 << 17]byte +var z15463 [1 << 17]byte +var z15464 [1 << 17]byte +var z15465 [1 << 17]byte +var z15466 [1 << 17]byte +var z15467 [1 << 17]byte +var z15468 [1 << 17]byte +var z15469 [1 << 17]byte +var z15470 [1 << 17]byte +var z15471 [1 << 17]byte +var z15472 [1 << 17]byte +var z15473 [1 << 17]byte +var z15474 [1 << 17]byte +var z15475 [1 << 17]byte +var z15476 [1 << 17]byte +var z15477 [1 << 17]byte +var z15478 [1 << 17]byte +var z15479 [1 << 17]byte +var z15480 [1 << 17]byte +var z15481 [1 << 17]byte +var z15482 [1 << 17]byte +var z15483 [1 << 17]byte +var z15484 [1 << 17]byte +var z15485 [1 << 17]byte +var z15486 [1 << 17]byte +var z15487 [1 << 17]byte +var z15488 [1 << 17]byte +var z15489 [1 << 17]byte +var z15490 [1 << 17]byte +var z15491 [1 << 17]byte +var z15492 [1 << 17]byte +var z15493 [1 << 17]byte +var z15494 [1 << 17]byte +var z15495 [1 << 17]byte +var z15496 [1 << 17]byte +var z15497 [1 << 17]byte +var z15498 [1 << 17]byte +var z15499 [1 << 17]byte +var z15500 [1 << 17]byte +var z15501 [1 << 17]byte +var z15502 [1 << 17]byte +var z15503 [1 << 17]byte +var z15504 [1 << 17]byte +var z15505 [1 << 17]byte +var z15506 [1 << 17]byte +var z15507 [1 << 17]byte +var z15508 [1 << 17]byte +var z15509 [1 << 17]byte +var z15510 [1 << 17]byte +var z15511 [1 << 17]byte +var z15512 [1 << 17]byte +var z15513 [1 << 17]byte +var z15514 [1 << 17]byte +var z15515 [1 << 17]byte +var z15516 [1 << 17]byte +var z15517 [1 << 17]byte +var z15518 [1 << 17]byte +var z15519 [1 << 17]byte +var z15520 [1 << 17]byte +var z15521 [1 << 17]byte +var z15522 [1 << 17]byte +var z15523 [1 << 17]byte +var z15524 [1 << 17]byte +var z15525 [1 << 17]byte +var z15526 [1 << 17]byte +var z15527 [1 << 17]byte +var z15528 [1 << 17]byte +var z15529 [1 << 17]byte +var z15530 [1 << 17]byte +var z15531 [1 << 17]byte +var z15532 [1 << 17]byte +var z15533 [1 << 17]byte +var z15534 [1 << 17]byte +var z15535 [1 << 17]byte +var z15536 [1 << 17]byte +var z15537 [1 << 17]byte +var z15538 [1 << 17]byte +var z15539 [1 << 17]byte +var z15540 [1 << 17]byte +var z15541 [1 << 17]byte +var z15542 [1 << 17]byte +var z15543 [1 << 17]byte +var z15544 [1 << 17]byte +var z15545 [1 << 17]byte +var z15546 [1 << 17]byte +var z15547 [1 << 17]byte +var z15548 [1 << 17]byte +var z15549 [1 << 17]byte +var z15550 [1 << 17]byte +var z15551 [1 << 17]byte +var z15552 [1 << 17]byte +var z15553 [1 << 17]byte +var z15554 [1 << 17]byte +var z15555 [1 << 17]byte +var z15556 [1 << 17]byte +var z15557 [1 << 17]byte +var z15558 [1 << 17]byte +var z15559 [1 << 17]byte +var z15560 [1 << 17]byte +var z15561 [1 << 17]byte +var z15562 [1 << 17]byte +var z15563 [1 << 17]byte +var z15564 [1 << 17]byte +var z15565 [1 << 17]byte +var z15566 [1 << 17]byte +var z15567 [1 << 17]byte +var z15568 [1 << 17]byte +var z15569 [1 << 17]byte +var z15570 [1 << 17]byte +var z15571 [1 << 17]byte +var z15572 [1 << 17]byte +var z15573 [1 << 17]byte +var z15574 [1 << 17]byte +var z15575 [1 << 17]byte +var z15576 [1 << 17]byte +var z15577 [1 << 17]byte +var z15578 [1 << 17]byte +var z15579 [1 << 17]byte +var z15580 [1 << 17]byte +var z15581 [1 << 17]byte +var z15582 [1 << 17]byte +var z15583 [1 << 17]byte +var z15584 [1 << 17]byte +var z15585 [1 << 17]byte +var z15586 [1 << 17]byte +var z15587 [1 << 17]byte +var z15588 [1 << 17]byte +var z15589 [1 << 17]byte +var z15590 [1 << 17]byte +var z15591 [1 << 17]byte +var z15592 [1 << 17]byte +var z15593 [1 << 17]byte +var z15594 [1 << 17]byte +var z15595 [1 << 17]byte +var z15596 [1 << 17]byte +var z15597 [1 << 17]byte +var z15598 [1 << 17]byte +var z15599 [1 << 17]byte +var z15600 [1 << 17]byte +var z15601 [1 << 17]byte +var z15602 [1 << 17]byte +var z15603 [1 << 17]byte +var z15604 [1 << 17]byte +var z15605 [1 << 17]byte +var z15606 [1 << 17]byte +var z15607 [1 << 17]byte +var z15608 [1 << 17]byte +var z15609 [1 << 17]byte +var z15610 [1 << 17]byte +var z15611 [1 << 17]byte +var z15612 [1 << 17]byte +var z15613 [1 << 17]byte +var z15614 [1 << 17]byte +var z15615 [1 << 17]byte +var z15616 [1 << 17]byte +var z15617 [1 << 17]byte +var z15618 [1 << 17]byte +var z15619 [1 << 17]byte +var z15620 [1 << 17]byte +var z15621 [1 << 17]byte +var z15622 [1 << 17]byte +var z15623 [1 << 17]byte +var z15624 [1 << 17]byte +var z15625 [1 << 17]byte +var z15626 [1 << 17]byte +var z15627 [1 << 17]byte +var z15628 [1 << 17]byte +var z15629 [1 << 17]byte +var z15630 [1 << 17]byte +var z15631 [1 << 17]byte +var z15632 [1 << 17]byte +var z15633 [1 << 17]byte +var z15634 [1 << 17]byte +var z15635 [1 << 17]byte +var z15636 [1 << 17]byte +var z15637 [1 << 17]byte +var z15638 [1 << 17]byte +var z15639 [1 << 17]byte +var z15640 [1 << 17]byte +var z15641 [1 << 17]byte +var z15642 [1 << 17]byte +var z15643 [1 << 17]byte +var z15644 [1 << 17]byte +var z15645 [1 << 17]byte +var z15646 [1 << 17]byte +var z15647 [1 << 17]byte +var z15648 [1 << 17]byte +var z15649 [1 << 17]byte +var z15650 [1 << 17]byte +var z15651 [1 << 17]byte +var z15652 [1 << 17]byte +var z15653 [1 << 17]byte +var z15654 [1 << 17]byte +var z15655 [1 << 17]byte +var z15656 [1 << 17]byte +var z15657 [1 << 17]byte +var z15658 [1 << 17]byte +var z15659 [1 << 17]byte +var z15660 [1 << 17]byte +var z15661 [1 << 17]byte +var z15662 [1 << 17]byte +var z15663 [1 << 17]byte +var z15664 [1 << 17]byte +var z15665 [1 << 17]byte +var z15666 [1 << 17]byte +var z15667 [1 << 17]byte +var z15668 [1 << 17]byte +var z15669 [1 << 17]byte +var z15670 [1 << 17]byte +var z15671 [1 << 17]byte +var z15672 [1 << 17]byte +var z15673 [1 << 17]byte +var z15674 [1 << 17]byte +var z15675 [1 << 17]byte +var z15676 [1 << 17]byte +var z15677 [1 << 17]byte +var z15678 [1 << 17]byte +var z15679 [1 << 17]byte +var z15680 [1 << 17]byte +var z15681 [1 << 17]byte +var z15682 [1 << 17]byte +var z15683 [1 << 17]byte +var z15684 [1 << 17]byte +var z15685 [1 << 17]byte +var z15686 [1 << 17]byte +var z15687 [1 << 17]byte +var z15688 [1 << 17]byte +var z15689 [1 << 17]byte +var z15690 [1 << 17]byte +var z15691 [1 << 17]byte +var z15692 [1 << 17]byte +var z15693 [1 << 17]byte +var z15694 [1 << 17]byte +var z15695 [1 << 17]byte +var z15696 [1 << 17]byte +var z15697 [1 << 17]byte +var z15698 [1 << 17]byte +var z15699 [1 << 17]byte +var z15700 [1 << 17]byte +var z15701 [1 << 17]byte +var z15702 [1 << 17]byte +var z15703 [1 << 17]byte +var z15704 [1 << 17]byte +var z15705 [1 << 17]byte +var z15706 [1 << 17]byte +var z15707 [1 << 17]byte +var z15708 [1 << 17]byte +var z15709 [1 << 17]byte +var z15710 [1 << 17]byte +var z15711 [1 << 17]byte +var z15712 [1 << 17]byte +var z15713 [1 << 17]byte +var z15714 [1 << 17]byte +var z15715 [1 << 17]byte +var z15716 [1 << 17]byte +var z15717 [1 << 17]byte +var z15718 [1 << 17]byte +var z15719 [1 << 17]byte +var z15720 [1 << 17]byte +var z15721 [1 << 17]byte +var z15722 [1 << 17]byte +var z15723 [1 << 17]byte +var z15724 [1 << 17]byte +var z15725 [1 << 17]byte +var z15726 [1 << 17]byte +var z15727 [1 << 17]byte +var z15728 [1 << 17]byte +var z15729 [1 << 17]byte +var z15730 [1 << 17]byte +var z15731 [1 << 17]byte +var z15732 [1 << 17]byte +var z15733 [1 << 17]byte +var z15734 [1 << 17]byte +var z15735 [1 << 17]byte +var z15736 [1 << 17]byte +var z15737 [1 << 17]byte +var z15738 [1 << 17]byte +var z15739 [1 << 17]byte +var z15740 [1 << 17]byte +var z15741 [1 << 17]byte +var z15742 [1 << 17]byte +var z15743 [1 << 17]byte +var z15744 [1 << 17]byte +var z15745 [1 << 17]byte +var z15746 [1 << 17]byte +var z15747 [1 << 17]byte +var z15748 [1 << 17]byte +var z15749 [1 << 17]byte +var z15750 [1 << 17]byte +var z15751 [1 << 17]byte +var z15752 [1 << 17]byte +var z15753 [1 << 17]byte +var z15754 [1 << 17]byte +var z15755 [1 << 17]byte +var z15756 [1 << 17]byte +var z15757 [1 << 17]byte +var z15758 [1 << 17]byte +var z15759 [1 << 17]byte +var z15760 [1 << 17]byte +var z15761 [1 << 17]byte +var z15762 [1 << 17]byte +var z15763 [1 << 17]byte +var z15764 [1 << 17]byte +var z15765 [1 << 17]byte +var z15766 [1 << 17]byte +var z15767 [1 << 17]byte +var z15768 [1 << 17]byte +var z15769 [1 << 17]byte +var z15770 [1 << 17]byte +var z15771 [1 << 17]byte +var z15772 [1 << 17]byte +var z15773 [1 << 17]byte +var z15774 [1 << 17]byte +var z15775 [1 << 17]byte +var z15776 [1 << 17]byte +var z15777 [1 << 17]byte +var z15778 [1 << 17]byte +var z15779 [1 << 17]byte +var z15780 [1 << 17]byte +var z15781 [1 << 17]byte +var z15782 [1 << 17]byte +var z15783 [1 << 17]byte +var z15784 [1 << 17]byte +var z15785 [1 << 17]byte +var z15786 [1 << 17]byte +var z15787 [1 << 17]byte +var z15788 [1 << 17]byte +var z15789 [1 << 17]byte +var z15790 [1 << 17]byte +var z15791 [1 << 17]byte +var z15792 [1 << 17]byte +var z15793 [1 << 17]byte +var z15794 [1 << 17]byte +var z15795 [1 << 17]byte +var z15796 [1 << 17]byte +var z15797 [1 << 17]byte +var z15798 [1 << 17]byte +var z15799 [1 << 17]byte +var z15800 [1 << 17]byte +var z15801 [1 << 17]byte +var z15802 [1 << 17]byte +var z15803 [1 << 17]byte +var z15804 [1 << 17]byte +var z15805 [1 << 17]byte +var z15806 [1 << 17]byte +var z15807 [1 << 17]byte +var z15808 [1 << 17]byte +var z15809 [1 << 17]byte +var z15810 [1 << 17]byte +var z15811 [1 << 17]byte +var z15812 [1 << 17]byte +var z15813 [1 << 17]byte +var z15814 [1 << 17]byte +var z15815 [1 << 17]byte +var z15816 [1 << 17]byte +var z15817 [1 << 17]byte +var z15818 [1 << 17]byte +var z15819 [1 << 17]byte +var z15820 [1 << 17]byte +var z15821 [1 << 17]byte +var z15822 [1 << 17]byte +var z15823 [1 << 17]byte +var z15824 [1 << 17]byte +var z15825 [1 << 17]byte +var z15826 [1 << 17]byte +var z15827 [1 << 17]byte +var z15828 [1 << 17]byte +var z15829 [1 << 17]byte +var z15830 [1 << 17]byte +var z15831 [1 << 17]byte +var z15832 [1 << 17]byte +var z15833 [1 << 17]byte +var z15834 [1 << 17]byte +var z15835 [1 << 17]byte +var z15836 [1 << 17]byte +var z15837 [1 << 17]byte +var z15838 [1 << 17]byte +var z15839 [1 << 17]byte +var z15840 [1 << 17]byte +var z15841 [1 << 17]byte +var z15842 [1 << 17]byte +var z15843 [1 << 17]byte +var z15844 [1 << 17]byte +var z15845 [1 << 17]byte +var z15846 [1 << 17]byte +var z15847 [1 << 17]byte +var z15848 [1 << 17]byte +var z15849 [1 << 17]byte +var z15850 [1 << 17]byte +var z15851 [1 << 17]byte +var z15852 [1 << 17]byte +var z15853 [1 << 17]byte +var z15854 [1 << 17]byte +var z15855 [1 << 17]byte +var z15856 [1 << 17]byte +var z15857 [1 << 17]byte +var z15858 [1 << 17]byte +var z15859 [1 << 17]byte +var z15860 [1 << 17]byte +var z15861 [1 << 17]byte +var z15862 [1 << 17]byte +var z15863 [1 << 17]byte +var z15864 [1 << 17]byte +var z15865 [1 << 17]byte +var z15866 [1 << 17]byte +var z15867 [1 << 17]byte +var z15868 [1 << 17]byte +var z15869 [1 << 17]byte +var z15870 [1 << 17]byte +var z15871 [1 << 17]byte +var z15872 [1 << 17]byte +var z15873 [1 << 17]byte +var z15874 [1 << 17]byte +var z15875 [1 << 17]byte +var z15876 [1 << 17]byte +var z15877 [1 << 17]byte +var z15878 [1 << 17]byte +var z15879 [1 << 17]byte +var z15880 [1 << 17]byte +var z15881 [1 << 17]byte +var z15882 [1 << 17]byte +var z15883 [1 << 17]byte +var z15884 [1 << 17]byte +var z15885 [1 << 17]byte +var z15886 [1 << 17]byte +var z15887 [1 << 17]byte +var z15888 [1 << 17]byte +var z15889 [1 << 17]byte +var z15890 [1 << 17]byte +var z15891 [1 << 17]byte +var z15892 [1 << 17]byte +var z15893 [1 << 17]byte +var z15894 [1 << 17]byte +var z15895 [1 << 17]byte +var z15896 [1 << 17]byte +var z15897 [1 << 17]byte +var z15898 [1 << 17]byte +var z15899 [1 << 17]byte +var z15900 [1 << 17]byte +var z15901 [1 << 17]byte +var z15902 [1 << 17]byte +var z15903 [1 << 17]byte +var z15904 [1 << 17]byte +var z15905 [1 << 17]byte +var z15906 [1 << 17]byte +var z15907 [1 << 17]byte +var z15908 [1 << 17]byte +var z15909 [1 << 17]byte +var z15910 [1 << 17]byte +var z15911 [1 << 17]byte +var z15912 [1 << 17]byte +var z15913 [1 << 17]byte +var z15914 [1 << 17]byte +var z15915 [1 << 17]byte +var z15916 [1 << 17]byte +var z15917 [1 << 17]byte +var z15918 [1 << 17]byte +var z15919 [1 << 17]byte +var z15920 [1 << 17]byte +var z15921 [1 << 17]byte +var z15922 [1 << 17]byte +var z15923 [1 << 17]byte +var z15924 [1 << 17]byte +var z15925 [1 << 17]byte +var z15926 [1 << 17]byte +var z15927 [1 << 17]byte +var z15928 [1 << 17]byte +var z15929 [1 << 17]byte +var z15930 [1 << 17]byte +var z15931 [1 << 17]byte +var z15932 [1 << 17]byte +var z15933 [1 << 17]byte +var z15934 [1 << 17]byte +var z15935 [1 << 17]byte +var z15936 [1 << 17]byte +var z15937 [1 << 17]byte +var z15938 [1 << 17]byte +var z15939 [1 << 17]byte +var z15940 [1 << 17]byte +var z15941 [1 << 17]byte +var z15942 [1 << 17]byte +var z15943 [1 << 17]byte +var z15944 [1 << 17]byte +var z15945 [1 << 17]byte +var z15946 [1 << 17]byte +var z15947 [1 << 17]byte +var z15948 [1 << 17]byte +var z15949 [1 << 17]byte +var z15950 [1 << 17]byte +var z15951 [1 << 17]byte +var z15952 [1 << 17]byte +var z15953 [1 << 17]byte +var z15954 [1 << 17]byte +var z15955 [1 << 17]byte +var z15956 [1 << 17]byte +var z15957 [1 << 17]byte +var z15958 [1 << 17]byte +var z15959 [1 << 17]byte +var z15960 [1 << 17]byte +var z15961 [1 << 17]byte +var z15962 [1 << 17]byte +var z15963 [1 << 17]byte +var z15964 [1 << 17]byte +var z15965 [1 << 17]byte +var z15966 [1 << 17]byte +var z15967 [1 << 17]byte +var z15968 [1 << 17]byte +var z15969 [1 << 17]byte +var z15970 [1 << 17]byte +var z15971 [1 << 17]byte +var z15972 [1 << 17]byte +var z15973 [1 << 17]byte +var z15974 [1 << 17]byte +var z15975 [1 << 17]byte +var z15976 [1 << 17]byte +var z15977 [1 << 17]byte +var z15978 [1 << 17]byte +var z15979 [1 << 17]byte +var z15980 [1 << 17]byte +var z15981 [1 << 17]byte +var z15982 [1 << 17]byte +var z15983 [1 << 17]byte +var z15984 [1 << 17]byte +var z15985 [1 << 17]byte +var z15986 [1 << 17]byte +var z15987 [1 << 17]byte +var z15988 [1 << 17]byte +var z15989 [1 << 17]byte +var z15990 [1 << 17]byte +var z15991 [1 << 17]byte +var z15992 [1 << 17]byte +var z15993 [1 << 17]byte +var z15994 [1 << 17]byte +var z15995 [1 << 17]byte +var z15996 [1 << 17]byte +var z15997 [1 << 17]byte +var z15998 [1 << 17]byte +var z15999 [1 << 17]byte +var z16000 [1 << 17]byte +var z16001 [1 << 17]byte +var z16002 [1 << 17]byte +var z16003 [1 << 17]byte +var z16004 [1 << 17]byte +var z16005 [1 << 17]byte +var z16006 [1 << 17]byte +var z16007 [1 << 17]byte +var z16008 [1 << 17]byte +var z16009 [1 << 17]byte +var z16010 [1 << 17]byte +var z16011 [1 << 17]byte +var z16012 [1 << 17]byte +var z16013 [1 << 17]byte +var z16014 [1 << 17]byte +var z16015 [1 << 17]byte +var z16016 [1 << 17]byte +var z16017 [1 << 17]byte +var z16018 [1 << 17]byte +var z16019 [1 << 17]byte +var z16020 [1 << 17]byte +var z16021 [1 << 17]byte +var z16022 [1 << 17]byte +var z16023 [1 << 17]byte +var z16024 [1 << 17]byte +var z16025 [1 << 17]byte +var z16026 [1 << 17]byte +var z16027 [1 << 17]byte +var z16028 [1 << 17]byte +var z16029 [1 << 17]byte +var z16030 [1 << 17]byte +var z16031 [1 << 17]byte +var z16032 [1 << 17]byte +var z16033 [1 << 17]byte +var z16034 [1 << 17]byte +var z16035 [1 << 17]byte +var z16036 [1 << 17]byte +var z16037 [1 << 17]byte +var z16038 [1 << 17]byte +var z16039 [1 << 17]byte +var z16040 [1 << 17]byte +var z16041 [1 << 17]byte +var z16042 [1 << 17]byte +var z16043 [1 << 17]byte +var z16044 [1 << 17]byte +var z16045 [1 << 17]byte +var z16046 [1 << 17]byte +var z16047 [1 << 17]byte +var z16048 [1 << 17]byte +var z16049 [1 << 17]byte +var z16050 [1 << 17]byte +var z16051 [1 << 17]byte +var z16052 [1 << 17]byte +var z16053 [1 << 17]byte +var z16054 [1 << 17]byte +var z16055 [1 << 17]byte +var z16056 [1 << 17]byte +var z16057 [1 << 17]byte +var z16058 [1 << 17]byte +var z16059 [1 << 17]byte +var z16060 [1 << 17]byte +var z16061 [1 << 17]byte +var z16062 [1 << 17]byte +var z16063 [1 << 17]byte +var z16064 [1 << 17]byte +var z16065 [1 << 17]byte +var z16066 [1 << 17]byte +var z16067 [1 << 17]byte +var z16068 [1 << 17]byte +var z16069 [1 << 17]byte +var z16070 [1 << 17]byte +var z16071 [1 << 17]byte +var z16072 [1 << 17]byte +var z16073 [1 << 17]byte +var z16074 [1 << 17]byte +var z16075 [1 << 17]byte +var z16076 [1 << 17]byte +var z16077 [1 << 17]byte +var z16078 [1 << 17]byte +var z16079 [1 << 17]byte +var z16080 [1 << 17]byte +var z16081 [1 << 17]byte +var z16082 [1 << 17]byte +var z16083 [1 << 17]byte +var z16084 [1 << 17]byte +var z16085 [1 << 17]byte +var z16086 [1 << 17]byte +var z16087 [1 << 17]byte +var z16088 [1 << 17]byte +var z16089 [1 << 17]byte +var z16090 [1 << 17]byte +var z16091 [1 << 17]byte +var z16092 [1 << 17]byte +var z16093 [1 << 17]byte +var z16094 [1 << 17]byte +var z16095 [1 << 17]byte +var z16096 [1 << 17]byte +var z16097 [1 << 17]byte +var z16098 [1 << 17]byte +var z16099 [1 << 17]byte +var z16100 [1 << 17]byte +var z16101 [1 << 17]byte +var z16102 [1 << 17]byte +var z16103 [1 << 17]byte +var z16104 [1 << 17]byte +var z16105 [1 << 17]byte +var z16106 [1 << 17]byte +var z16107 [1 << 17]byte +var z16108 [1 << 17]byte +var z16109 [1 << 17]byte +var z16110 [1 << 17]byte +var z16111 [1 << 17]byte +var z16112 [1 << 17]byte +var z16113 [1 << 17]byte +var z16114 [1 << 17]byte +var z16115 [1 << 17]byte +var z16116 [1 << 17]byte +var z16117 [1 << 17]byte +var z16118 [1 << 17]byte +var z16119 [1 << 17]byte +var z16120 [1 << 17]byte +var z16121 [1 << 17]byte +var z16122 [1 << 17]byte +var z16123 [1 << 17]byte +var z16124 [1 << 17]byte +var z16125 [1 << 17]byte +var z16126 [1 << 17]byte +var z16127 [1 << 17]byte +var z16128 [1 << 17]byte +var z16129 [1 << 17]byte +var z16130 [1 << 17]byte +var z16131 [1 << 17]byte +var z16132 [1 << 17]byte +var z16133 [1 << 17]byte +var z16134 [1 << 17]byte +var z16135 [1 << 17]byte +var z16136 [1 << 17]byte +var z16137 [1 << 17]byte +var z16138 [1 << 17]byte +var z16139 [1 << 17]byte +var z16140 [1 << 17]byte +var z16141 [1 << 17]byte +var z16142 [1 << 17]byte +var z16143 [1 << 17]byte +var z16144 [1 << 17]byte +var z16145 [1 << 17]byte +var z16146 [1 << 17]byte +var z16147 [1 << 17]byte +var z16148 [1 << 17]byte +var z16149 [1 << 17]byte +var z16150 [1 << 17]byte +var z16151 [1 << 17]byte +var z16152 [1 << 17]byte +var z16153 [1 << 17]byte +var z16154 [1 << 17]byte +var z16155 [1 << 17]byte +var z16156 [1 << 17]byte +var z16157 [1 << 17]byte +var z16158 [1 << 17]byte +var z16159 [1 << 17]byte +var z16160 [1 << 17]byte +var z16161 [1 << 17]byte +var z16162 [1 << 17]byte +var z16163 [1 << 17]byte +var z16164 [1 << 17]byte +var z16165 [1 << 17]byte +var z16166 [1 << 17]byte +var z16167 [1 << 17]byte +var z16168 [1 << 17]byte +var z16169 [1 << 17]byte +var z16170 [1 << 17]byte +var z16171 [1 << 17]byte +var z16172 [1 << 17]byte +var z16173 [1 << 17]byte +var z16174 [1 << 17]byte +var z16175 [1 << 17]byte +var z16176 [1 << 17]byte +var z16177 [1 << 17]byte +var z16178 [1 << 17]byte +var z16179 [1 << 17]byte +var z16180 [1 << 17]byte +var z16181 [1 << 17]byte +var z16182 [1 << 17]byte +var z16183 [1 << 17]byte +var z16184 [1 << 17]byte +var z16185 [1 << 17]byte +var z16186 [1 << 17]byte +var z16187 [1 << 17]byte +var z16188 [1 << 17]byte +var z16189 [1 << 17]byte +var z16190 [1 << 17]byte +var z16191 [1 << 17]byte +var z16192 [1 << 17]byte +var z16193 [1 << 17]byte +var z16194 [1 << 17]byte +var z16195 [1 << 17]byte +var z16196 [1 << 17]byte +var z16197 [1 << 17]byte +var z16198 [1 << 17]byte +var z16199 [1 << 17]byte +var z16200 [1 << 17]byte +var z16201 [1 << 17]byte +var z16202 [1 << 17]byte +var z16203 [1 << 17]byte +var z16204 [1 << 17]byte +var z16205 [1 << 17]byte +var z16206 [1 << 17]byte +var z16207 [1 << 17]byte +var z16208 [1 << 17]byte +var z16209 [1 << 17]byte +var z16210 [1 << 17]byte +var z16211 [1 << 17]byte +var z16212 [1 << 17]byte +var z16213 [1 << 17]byte +var z16214 [1 << 17]byte +var z16215 [1 << 17]byte +var z16216 [1 << 17]byte +var z16217 [1 << 17]byte +var z16218 [1 << 17]byte +var z16219 [1 << 17]byte +var z16220 [1 << 17]byte +var z16221 [1 << 17]byte +var z16222 [1 << 17]byte +var z16223 [1 << 17]byte +var z16224 [1 << 17]byte +var z16225 [1 << 17]byte +var z16226 [1 << 17]byte +var z16227 [1 << 17]byte +var z16228 [1 << 17]byte +var z16229 [1 << 17]byte +var z16230 [1 << 17]byte +var z16231 [1 << 17]byte +var z16232 [1 << 17]byte +var z16233 [1 << 17]byte +var z16234 [1 << 17]byte +var z16235 [1 << 17]byte +var z16236 [1 << 17]byte +var z16237 [1 << 17]byte +var z16238 [1 << 17]byte +var z16239 [1 << 17]byte +var z16240 [1 << 17]byte +var z16241 [1 << 17]byte +var z16242 [1 << 17]byte +var z16243 [1 << 17]byte +var z16244 [1 << 17]byte +var z16245 [1 << 17]byte +var z16246 [1 << 17]byte +var z16247 [1 << 17]byte +var z16248 [1 << 17]byte +var z16249 [1 << 17]byte +var z16250 [1 << 17]byte +var z16251 [1 << 17]byte +var z16252 [1 << 17]byte +var z16253 [1 << 17]byte +var z16254 [1 << 17]byte +var z16255 [1 << 17]byte +var z16256 [1 << 17]byte +var z16257 [1 << 17]byte +var z16258 [1 << 17]byte +var z16259 [1 << 17]byte +var z16260 [1 << 17]byte +var z16261 [1 << 17]byte +var z16262 [1 << 17]byte +var z16263 [1 << 17]byte +var z16264 [1 << 17]byte +var z16265 [1 << 17]byte +var z16266 [1 << 17]byte +var z16267 [1 << 17]byte +var z16268 [1 << 17]byte +var z16269 [1 << 17]byte +var z16270 [1 << 17]byte +var z16271 [1 << 17]byte +var z16272 [1 << 17]byte +var z16273 [1 << 17]byte +var z16274 [1 << 17]byte +var z16275 [1 << 17]byte +var z16276 [1 << 17]byte +var z16277 [1 << 17]byte +var z16278 [1 << 17]byte +var z16279 [1 << 17]byte +var z16280 [1 << 17]byte +var z16281 [1 << 17]byte +var z16282 [1 << 17]byte +var z16283 [1 << 17]byte +var z16284 [1 << 17]byte +var z16285 [1 << 17]byte +var z16286 [1 << 17]byte +var z16287 [1 << 17]byte +var z16288 [1 << 17]byte +var z16289 [1 << 17]byte +var z16290 [1 << 17]byte +var z16291 [1 << 17]byte +var z16292 [1 << 17]byte +var z16293 [1 << 17]byte +var z16294 [1 << 17]byte +var z16295 [1 << 17]byte +var z16296 [1 << 17]byte +var z16297 [1 << 17]byte +var z16298 [1 << 17]byte +var z16299 [1 << 17]byte +var z16300 [1 << 17]byte +var z16301 [1 << 17]byte +var z16302 [1 << 17]byte +var z16303 [1 << 17]byte +var z16304 [1 << 17]byte +var z16305 [1 << 17]byte +var z16306 [1 << 17]byte +var z16307 [1 << 17]byte +var z16308 [1 << 17]byte +var z16309 [1 << 17]byte +var z16310 [1 << 17]byte +var z16311 [1 << 17]byte +var z16312 [1 << 17]byte +var z16313 [1 << 17]byte +var z16314 [1 << 17]byte +var z16315 [1 << 17]byte +var z16316 [1 << 17]byte +var z16317 [1 << 17]byte +var z16318 [1 << 17]byte +var z16319 [1 << 17]byte +var z16320 [1 << 17]byte +var z16321 [1 << 17]byte +var z16322 [1 << 17]byte +var z16323 [1 << 17]byte +var z16324 [1 << 17]byte +var z16325 [1 << 17]byte +var z16326 [1 << 17]byte +var z16327 [1 << 17]byte +var z16328 [1 << 17]byte +var z16329 [1 << 17]byte +var z16330 [1 << 17]byte +var z16331 [1 << 17]byte +var z16332 [1 << 17]byte +var z16333 [1 << 17]byte +var z16334 [1 << 17]byte +var z16335 [1 << 17]byte +var z16336 [1 << 17]byte +var z16337 [1 << 17]byte +var z16338 [1 << 17]byte +var z16339 [1 << 17]byte +var z16340 [1 << 17]byte +var z16341 [1 << 17]byte +var z16342 [1 << 17]byte +var z16343 [1 << 17]byte +var z16344 [1 << 17]byte +var z16345 [1 << 17]byte +var z16346 [1 << 17]byte +var z16347 [1 << 17]byte +var z16348 [1 << 17]byte +var z16349 [1 << 17]byte +var z16350 [1 << 17]byte +var z16351 [1 << 17]byte +var z16352 [1 << 17]byte +var z16353 [1 << 17]byte +var z16354 [1 << 17]byte +var z16355 [1 << 17]byte +var z16356 [1 << 17]byte +var z16357 [1 << 17]byte +var z16358 [1 << 17]byte +var z16359 [1 << 17]byte +var z16360 [1 << 17]byte +var z16361 [1 << 17]byte +var z16362 [1 << 17]byte +var z16363 [1 << 17]byte +var z16364 [1 << 17]byte +var z16365 [1 << 17]byte +var z16366 [1 << 17]byte +var z16367 [1 << 17]byte +var z16368 [1 << 17]byte +var z16369 [1 << 17]byte +var z16370 [1 << 17]byte +var z16371 [1 << 17]byte +var z16372 [1 << 17]byte +var z16373 [1 << 17]byte +var z16374 [1 << 17]byte +var z16375 [1 << 17]byte +var z16376 [1 << 17]byte +var z16377 [1 << 17]byte +var z16378 [1 << 17]byte +var z16379 [1 << 17]byte +var z16380 [1 << 17]byte +var z16381 [1 << 17]byte +var z16382 [1 << 17]byte +var z16383 [1 << 17]byte +var z16384 [1 << 17]byte +var z16385 [1 << 17]byte +var z16386 [1 << 17]byte +var z16387 [1 << 17]byte +var z16388 [1 << 17]byte +var z16389 [1 << 17]byte +var z16390 [1 << 17]byte +var z16391 [1 << 17]byte +var z16392 [1 << 17]byte +var z16393 [1 << 17]byte +var z16394 [1 << 17]byte +var z16395 [1 << 17]byte +var z16396 [1 << 17]byte +var z16397 [1 << 17]byte +var z16398 [1 << 17]byte +var z16399 [1 << 17]byte +var z16400 [1 << 17]byte +var z16401 [1 << 17]byte +var z16402 [1 << 17]byte +var z16403 [1 << 17]byte +var z16404 [1 << 17]byte +var z16405 [1 << 17]byte +var z16406 [1 << 17]byte +var z16407 [1 << 17]byte +var z16408 [1 << 17]byte +var z16409 [1 << 17]byte +var z16410 [1 << 17]byte +var z16411 [1 << 17]byte +var z16412 [1 << 17]byte +var z16413 [1 << 17]byte +var z16414 [1 << 17]byte +var z16415 [1 << 17]byte +var z16416 [1 << 17]byte +var z16417 [1 << 17]byte +var z16418 [1 << 17]byte +var z16419 [1 << 17]byte +var z16420 [1 << 17]byte +var z16421 [1 << 17]byte +var z16422 [1 << 17]byte +var z16423 [1 << 17]byte +var z16424 [1 << 17]byte +var z16425 [1 << 17]byte +var z16426 [1 << 17]byte +var z16427 [1 << 17]byte +var z16428 [1 << 17]byte +var z16429 [1 << 17]byte +var z16430 [1 << 17]byte +var z16431 [1 << 17]byte +var z16432 [1 << 17]byte +var z16433 [1 << 17]byte +var z16434 [1 << 17]byte +var z16435 [1 << 17]byte +var z16436 [1 << 17]byte +var z16437 [1 << 17]byte +var z16438 [1 << 17]byte +var z16439 [1 << 17]byte +var z16440 [1 << 17]byte +var z16441 [1 << 17]byte +var z16442 [1 << 17]byte +var z16443 [1 << 17]byte +var z16444 [1 << 17]byte +var z16445 [1 << 17]byte +var z16446 [1 << 17]byte +var z16447 [1 << 17]byte +var z16448 [1 << 17]byte +var z16449 [1 << 17]byte +var z16450 [1 << 17]byte +var z16451 [1 << 17]byte +var z16452 [1 << 17]byte +var z16453 [1 << 17]byte +var z16454 [1 << 17]byte +var z16455 [1 << 17]byte +var z16456 [1 << 17]byte +var z16457 [1 << 17]byte +var z16458 [1 << 17]byte +var z16459 [1 << 17]byte +var z16460 [1 << 17]byte +var z16461 [1 << 17]byte +var z16462 [1 << 17]byte +var z16463 [1 << 17]byte +var z16464 [1 << 17]byte +var z16465 [1 << 17]byte +var z16466 [1 << 17]byte +var z16467 [1 << 17]byte +var z16468 [1 << 17]byte +var z16469 [1 << 17]byte +var z16470 [1 << 17]byte +var z16471 [1 << 17]byte +var z16472 [1 << 17]byte +var z16473 [1 << 17]byte +var z16474 [1 << 17]byte +var z16475 [1 << 17]byte +var z16476 [1 << 17]byte +var z16477 [1 << 17]byte +var z16478 [1 << 17]byte +var z16479 [1 << 17]byte +var z16480 [1 << 17]byte func main() { // GC_ERROR "stack frame too large" - // seq 1 206 | sed 's/.*/ var x& [10<<20]byte/' - // seq 1 206 | sed 's/.*/ z = x&/' - var x1 [10<<20]byte - var x2 [10<<20]byte - var x3 [10<<20]byte - var x4 [10<<20]byte - var x5 [10<<20]byte - var x6 [10<<20]byte - var x7 [10<<20]byte - var x8 [10<<20]byte - var x9 [10<<20]byte - var x10 [10<<20]byte - var x11 [10<<20]byte - var x12 [10<<20]byte - var x13 [10<<20]byte - var x14 [10<<20]byte - var x15 [10<<20]byte - var x16 [10<<20]byte - var x17 [10<<20]byte - var x18 [10<<20]byte - var x19 [10<<20]byte - var x20 [10<<20]byte - var x21 [10<<20]byte - var x22 [10<<20]byte - var x23 [10<<20]byte - var x24 [10<<20]byte - var x25 [10<<20]byte - var x26 [10<<20]byte - var x27 [10<<20]byte - var x28 [10<<20]byte - var x29 [10<<20]byte - var x30 [10<<20]byte - var x31 [10<<20]byte - var x32 [10<<20]byte - var x33 [10<<20]byte - var x34 [10<<20]byte - var x35 [10<<20]byte - var x36 [10<<20]byte - var x37 [10<<20]byte - var x38 [10<<20]byte - var x39 [10<<20]byte - var x40 [10<<20]byte - var x41 [10<<20]byte - var x42 [10<<20]byte - var x43 [10<<20]byte - var x44 [10<<20]byte - var x45 [10<<20]byte - var x46 [10<<20]byte - var x47 [10<<20]byte - var x48 [10<<20]byte - var x49 [10<<20]byte - var x50 [10<<20]byte - var x51 [10<<20]byte - var x52 [10<<20]byte - var x53 [10<<20]byte - var x54 [10<<20]byte - var x55 [10<<20]byte - var x56 [10<<20]byte - var x57 [10<<20]byte - var x58 [10<<20]byte - var x59 [10<<20]byte - var x60 [10<<20]byte - var x61 [10<<20]byte - var x62 [10<<20]byte - var x63 [10<<20]byte - var x64 [10<<20]byte - var x65 [10<<20]byte - var x66 [10<<20]byte - var x67 [10<<20]byte - var x68 [10<<20]byte - var x69 [10<<20]byte - var x70 [10<<20]byte - var x71 [10<<20]byte - var x72 [10<<20]byte - var x73 [10<<20]byte - var x74 [10<<20]byte - var x75 [10<<20]byte - var x76 [10<<20]byte - var x77 [10<<20]byte - var x78 [10<<20]byte - var x79 [10<<20]byte - var x80 [10<<20]byte - var x81 [10<<20]byte - var x82 [10<<20]byte - var x83 [10<<20]byte - var x84 [10<<20]byte - var x85 [10<<20]byte - var x86 [10<<20]byte - var x87 [10<<20]byte - var x88 [10<<20]byte - var x89 [10<<20]byte - var x90 [10<<20]byte - var x91 [10<<20]byte - var x92 [10<<20]byte - var x93 [10<<20]byte - var x94 [10<<20]byte - var x95 [10<<20]byte - var x96 [10<<20]byte - var x97 [10<<20]byte - var x98 [10<<20]byte - var x99 [10<<20]byte - var x100 [10<<20]byte - var x101 [10<<20]byte - var x102 [10<<20]byte - var x103 [10<<20]byte - var x104 [10<<20]byte - var x105 [10<<20]byte - var x106 [10<<20]byte - var x107 [10<<20]byte - var x108 [10<<20]byte - var x109 [10<<20]byte - var x110 [10<<20]byte - var x111 [10<<20]byte - var x112 [10<<20]byte - var x113 [10<<20]byte - var x114 [10<<20]byte - var x115 [10<<20]byte - var x116 [10<<20]byte - var x117 [10<<20]byte - var x118 [10<<20]byte - var x119 [10<<20]byte - var x120 [10<<20]byte - var x121 [10<<20]byte - var x122 [10<<20]byte - var x123 [10<<20]byte - var x124 [10<<20]byte - var x125 [10<<20]byte - var x126 [10<<20]byte - var x127 [10<<20]byte - var x128 [10<<20]byte - var x129 [10<<20]byte - var x130 [10<<20]byte - var x131 [10<<20]byte - var x132 [10<<20]byte - var x133 [10<<20]byte - var x134 [10<<20]byte - var x135 [10<<20]byte - var x136 [10<<20]byte - var x137 [10<<20]byte - var x138 [10<<20]byte - var x139 [10<<20]byte - var x140 [10<<20]byte - var x141 [10<<20]byte - var x142 [10<<20]byte - var x143 [10<<20]byte - var x144 [10<<20]byte - var x145 [10<<20]byte - var x146 [10<<20]byte - var x147 [10<<20]byte - var x148 [10<<20]byte - var x149 [10<<20]byte - var x150 [10<<20]byte - var x151 [10<<20]byte - var x152 [10<<20]byte - var x153 [10<<20]byte - var x154 [10<<20]byte - var x155 [10<<20]byte - var x156 [10<<20]byte - var x157 [10<<20]byte - var x158 [10<<20]byte - var x159 [10<<20]byte - var x160 [10<<20]byte - var x161 [10<<20]byte - var x162 [10<<20]byte - var x163 [10<<20]byte - var x164 [10<<20]byte - var x165 [10<<20]byte - var x166 [10<<20]byte - var x167 [10<<20]byte - var x168 [10<<20]byte - var x169 [10<<20]byte - var x170 [10<<20]byte - var x171 [10<<20]byte - var x172 [10<<20]byte - var x173 [10<<20]byte - var x174 [10<<20]byte - var x175 [10<<20]byte - var x176 [10<<20]byte - var x177 [10<<20]byte - var x178 [10<<20]byte - var x179 [10<<20]byte - var x180 [10<<20]byte - var x181 [10<<20]byte - var x182 [10<<20]byte - var x183 [10<<20]byte - var x184 [10<<20]byte - var x185 [10<<20]byte - var x186 [10<<20]byte - var x187 [10<<20]byte - var x188 [10<<20]byte - var x189 [10<<20]byte - var x190 [10<<20]byte - var x191 [10<<20]byte - var x192 [10<<20]byte - var x193 [10<<20]byte - var x194 [10<<20]byte - var x195 [10<<20]byte - var x196 [10<<20]byte - var x197 [10<<20]byte - var x198 [10<<20]byte - var x199 [10<<20]byte - var x200 [10<<20]byte - var x201 [10<<20]byte - var x202 [10<<20]byte - var x203 [10<<20]byte - var x204 [10<<20]byte - var x205 [10<<20]byte - var x206 [10<<20]byte - var x207 [10<<20]byte - z = x1 - z = x2 - z = x3 - z = x4 - z = x5 - z = x6 - z = x7 - z = x8 - z = x9 - z = x10 - z = x11 - z = x12 - z = x13 - z = x14 - z = x15 - z = x16 - z = x17 - z = x18 - z = x19 - z = x20 - z = x21 - z = x22 - z = x23 - z = x24 - z = x25 - z = x26 - z = x27 - z = x28 - z = x29 - z = x30 - z = x31 - z = x32 - z = x33 - z = x34 - z = x35 - z = x36 - z = x37 - z = x38 - z = x39 - z = x40 - z = x41 - z = x42 - z = x43 - z = x44 - z = x45 - z = x46 - z = x47 - z = x48 - z = x49 - z = x50 - z = x51 - z = x52 - z = x53 - z = x54 - z = x55 - z = x56 - z = x57 - z = x58 - z = x59 - z = x60 - z = x61 - z = x62 - z = x63 - z = x64 - z = x65 - z = x66 - z = x67 - z = x68 - z = x69 - z = x70 - z = x71 - z = x72 - z = x73 - z = x74 - z = x75 - z = x76 - z = x77 - z = x78 - z = x79 - z = x80 - z = x81 - z = x82 - z = x83 - z = x84 - z = x85 - z = x86 - z = x87 - z = x88 - z = x89 - z = x90 - z = x91 - z = x92 - z = x93 - z = x94 - z = x95 - z = x96 - z = x97 - z = x98 - z = x99 - z = x100 - z = x101 - z = x102 - z = x103 - z = x104 - z = x105 - z = x106 - z = x107 - z = x108 - z = x109 - z = x110 - z = x111 - z = x112 - z = x113 - z = x114 - z = x115 - z = x116 - z = x117 - z = x118 - z = x119 - z = x120 - z = x121 - z = x122 - z = x123 - z = x124 - z = x125 - z = x126 - z = x127 - z = x128 - z = x129 - z = x130 - z = x131 - z = x132 - z = x133 - z = x134 - z = x135 - z = x136 - z = x137 - z = x138 - z = x139 - z = x140 - z = x141 - z = x142 - z = x143 - z = x144 - z = x145 - z = x146 - z = x147 - z = x148 - z = x149 - z = x150 - z = x151 - z = x152 - z = x153 - z = x154 - z = x155 - z = x156 - z = x157 - z = x158 - z = x159 - z = x160 - z = x161 - z = x162 - z = x163 - z = x164 - z = x165 - z = x166 - z = x167 - z = x168 - z = x169 - z = x170 - z = x171 - z = x172 - z = x173 - z = x174 - z = x175 - z = x176 - z = x177 - z = x178 - z = x179 - z = x180 - z = x181 - z = x182 - z = x183 - z = x184 - z = x185 - z = x186 - z = x187 - z = x188 - z = x189 - z = x190 - z = x191 - z = x192 - z = x193 - z = x194 - z = x195 - z = x196 - z = x197 - z = x198 - z = x199 - z = x200 - z = x201 - z = x202 - z = x203 - z = x204 - z = x205 - z = x206 - z = x207 + // seq 1 16480 | sed 's/.*/ var x& [1 << 17]byte/' + // seq 1 16480 | sed 's/.*/ z& = x&/' + var x1 [1 << 17]byte + var x2 [1 << 17]byte + var x3 [1 << 17]byte + var x4 [1 << 17]byte + var x5 [1 << 17]byte + var x6 [1 << 17]byte + var x7 [1 << 17]byte + var x8 [1 << 17]byte + var x9 [1 << 17]byte + var x10 [1 << 17]byte + var x11 [1 << 17]byte + var x12 [1 << 17]byte + var x13 [1 << 17]byte + var x14 [1 << 17]byte + var x15 [1 << 17]byte + var x16 [1 << 17]byte + var x17 [1 << 17]byte + var x18 [1 << 17]byte + var x19 [1 << 17]byte + var x20 [1 << 17]byte + var x21 [1 << 17]byte + var x22 [1 << 17]byte + var x23 [1 << 17]byte + var x24 [1 << 17]byte + var x25 [1 << 17]byte + var x26 [1 << 17]byte + var x27 [1 << 17]byte + var x28 [1 << 17]byte + var x29 [1 << 17]byte + var x30 [1 << 17]byte + var x31 [1 << 17]byte + var x32 [1 << 17]byte + var x33 [1 << 17]byte + var x34 [1 << 17]byte + var x35 [1 << 17]byte + var x36 [1 << 17]byte + var x37 [1 << 17]byte + var x38 [1 << 17]byte + var x39 [1 << 17]byte + var x40 [1 << 17]byte + var x41 [1 << 17]byte + var x42 [1 << 17]byte + var x43 [1 << 17]byte + var x44 [1 << 17]byte + var x45 [1 << 17]byte + var x46 [1 << 17]byte + var x47 [1 << 17]byte + var x48 [1 << 17]byte + var x49 [1 << 17]byte + var x50 [1 << 17]byte + var x51 [1 << 17]byte + var x52 [1 << 17]byte + var x53 [1 << 17]byte + var x54 [1 << 17]byte + var x55 [1 << 17]byte + var x56 [1 << 17]byte + var x57 [1 << 17]byte + var x58 [1 << 17]byte + var x59 [1 << 17]byte + var x60 [1 << 17]byte + var x61 [1 << 17]byte + var x62 [1 << 17]byte + var x63 [1 << 17]byte + var x64 [1 << 17]byte + var x65 [1 << 17]byte + var x66 [1 << 17]byte + var x67 [1 << 17]byte + var x68 [1 << 17]byte + var x69 [1 << 17]byte + var x70 [1 << 17]byte + var x71 [1 << 17]byte + var x72 [1 << 17]byte + var x73 [1 << 17]byte + var x74 [1 << 17]byte + var x75 [1 << 17]byte + var x76 [1 << 17]byte + var x77 [1 << 17]byte + var x78 [1 << 17]byte + var x79 [1 << 17]byte + var x80 [1 << 17]byte + var x81 [1 << 17]byte + var x82 [1 << 17]byte + var x83 [1 << 17]byte + var x84 [1 << 17]byte + var x85 [1 << 17]byte + var x86 [1 << 17]byte + var x87 [1 << 17]byte + var x88 [1 << 17]byte + var x89 [1 << 17]byte + var x90 [1 << 17]byte + var x91 [1 << 17]byte + var x92 [1 << 17]byte + var x93 [1 << 17]byte + var x94 [1 << 17]byte + var x95 [1 << 17]byte + var x96 [1 << 17]byte + var x97 [1 << 17]byte + var x98 [1 << 17]byte + var x99 [1 << 17]byte + var x100 [1 << 17]byte + var x101 [1 << 17]byte + var x102 [1 << 17]byte + var x103 [1 << 17]byte + var x104 [1 << 17]byte + var x105 [1 << 17]byte + var x106 [1 << 17]byte + var x107 [1 << 17]byte + var x108 [1 << 17]byte + var x109 [1 << 17]byte + var x110 [1 << 17]byte + var x111 [1 << 17]byte + var x112 [1 << 17]byte + var x113 [1 << 17]byte + var x114 [1 << 17]byte + var x115 [1 << 17]byte + var x116 [1 << 17]byte + var x117 [1 << 17]byte + var x118 [1 << 17]byte + var x119 [1 << 17]byte + var x120 [1 << 17]byte + var x121 [1 << 17]byte + var x122 [1 << 17]byte + var x123 [1 << 17]byte + var x124 [1 << 17]byte + var x125 [1 << 17]byte + var x126 [1 << 17]byte + var x127 [1 << 17]byte + var x128 [1 << 17]byte + var x129 [1 << 17]byte + var x130 [1 << 17]byte + var x131 [1 << 17]byte + var x132 [1 << 17]byte + var x133 [1 << 17]byte + var x134 [1 << 17]byte + var x135 [1 << 17]byte + var x136 [1 << 17]byte + var x137 [1 << 17]byte + var x138 [1 << 17]byte + var x139 [1 << 17]byte + var x140 [1 << 17]byte + var x141 [1 << 17]byte + var x142 [1 << 17]byte + var x143 [1 << 17]byte + var x144 [1 << 17]byte + var x145 [1 << 17]byte + var x146 [1 << 17]byte + var x147 [1 << 17]byte + var x148 [1 << 17]byte + var x149 [1 << 17]byte + var x150 [1 << 17]byte + var x151 [1 << 17]byte + var x152 [1 << 17]byte + var x153 [1 << 17]byte + var x154 [1 << 17]byte + var x155 [1 << 17]byte + var x156 [1 << 17]byte + var x157 [1 << 17]byte + var x158 [1 << 17]byte + var x159 [1 << 17]byte + var x160 [1 << 17]byte + var x161 [1 << 17]byte + var x162 [1 << 17]byte + var x163 [1 << 17]byte + var x164 [1 << 17]byte + var x165 [1 << 17]byte + var x166 [1 << 17]byte + var x167 [1 << 17]byte + var x168 [1 << 17]byte + var x169 [1 << 17]byte + var x170 [1 << 17]byte + var x171 [1 << 17]byte + var x172 [1 << 17]byte + var x173 [1 << 17]byte + var x174 [1 << 17]byte + var x175 [1 << 17]byte + var x176 [1 << 17]byte + var x177 [1 << 17]byte + var x178 [1 << 17]byte + var x179 [1 << 17]byte + var x180 [1 << 17]byte + var x181 [1 << 17]byte + var x182 [1 << 17]byte + var x183 [1 << 17]byte + var x184 [1 << 17]byte + var x185 [1 << 17]byte + var x186 [1 << 17]byte + var x187 [1 << 17]byte + var x188 [1 << 17]byte + var x189 [1 << 17]byte + var x190 [1 << 17]byte + var x191 [1 << 17]byte + var x192 [1 << 17]byte + var x193 [1 << 17]byte + var x194 [1 << 17]byte + var x195 [1 << 17]byte + var x196 [1 << 17]byte + var x197 [1 << 17]byte + var x198 [1 << 17]byte + var x199 [1 << 17]byte + var x200 [1 << 17]byte + var x201 [1 << 17]byte + var x202 [1 << 17]byte + var x203 [1 << 17]byte + var x204 [1 << 17]byte + var x205 [1 << 17]byte + var x206 [1 << 17]byte + var x207 [1 << 17]byte + var x208 [1 << 17]byte + var x209 [1 << 17]byte + var x210 [1 << 17]byte + var x211 [1 << 17]byte + var x212 [1 << 17]byte + var x213 [1 << 17]byte + var x214 [1 << 17]byte + var x215 [1 << 17]byte + var x216 [1 << 17]byte + var x217 [1 << 17]byte + var x218 [1 << 17]byte + var x219 [1 << 17]byte + var x220 [1 << 17]byte + var x221 [1 << 17]byte + var x222 [1 << 17]byte + var x223 [1 << 17]byte + var x224 [1 << 17]byte + var x225 [1 << 17]byte + var x226 [1 << 17]byte + var x227 [1 << 17]byte + var x228 [1 << 17]byte + var x229 [1 << 17]byte + var x230 [1 << 17]byte + var x231 [1 << 17]byte + var x232 [1 << 17]byte + var x233 [1 << 17]byte + var x234 [1 << 17]byte + var x235 [1 << 17]byte + var x236 [1 << 17]byte + var x237 [1 << 17]byte + var x238 [1 << 17]byte + var x239 [1 << 17]byte + var x240 [1 << 17]byte + var x241 [1 << 17]byte + var x242 [1 << 17]byte + var x243 [1 << 17]byte + var x244 [1 << 17]byte + var x245 [1 << 17]byte + var x246 [1 << 17]byte + var x247 [1 << 17]byte + var x248 [1 << 17]byte + var x249 [1 << 17]byte + var x250 [1 << 17]byte + var x251 [1 << 17]byte + var x252 [1 << 17]byte + var x253 [1 << 17]byte + var x254 [1 << 17]byte + var x255 [1 << 17]byte + var x256 [1 << 17]byte + var x257 [1 << 17]byte + var x258 [1 << 17]byte + var x259 [1 << 17]byte + var x260 [1 << 17]byte + var x261 [1 << 17]byte + var x262 [1 << 17]byte + var x263 [1 << 17]byte + var x264 [1 << 17]byte + var x265 [1 << 17]byte + var x266 [1 << 17]byte + var x267 [1 << 17]byte + var x268 [1 << 17]byte + var x269 [1 << 17]byte + var x270 [1 << 17]byte + var x271 [1 << 17]byte + var x272 [1 << 17]byte + var x273 [1 << 17]byte + var x274 [1 << 17]byte + var x275 [1 << 17]byte + var x276 [1 << 17]byte + var x277 [1 << 17]byte + var x278 [1 << 17]byte + var x279 [1 << 17]byte + var x280 [1 << 17]byte + var x281 [1 << 17]byte + var x282 [1 << 17]byte + var x283 [1 << 17]byte + var x284 [1 << 17]byte + var x285 [1 << 17]byte + var x286 [1 << 17]byte + var x287 [1 << 17]byte + var x288 [1 << 17]byte + var x289 [1 << 17]byte + var x290 [1 << 17]byte + var x291 [1 << 17]byte + var x292 [1 << 17]byte + var x293 [1 << 17]byte + var x294 [1 << 17]byte + var x295 [1 << 17]byte + var x296 [1 << 17]byte + var x297 [1 << 17]byte + var x298 [1 << 17]byte + var x299 [1 << 17]byte + var x300 [1 << 17]byte + var x301 [1 << 17]byte + var x302 [1 << 17]byte + var x303 [1 << 17]byte + var x304 [1 << 17]byte + var x305 [1 << 17]byte + var x306 [1 << 17]byte + var x307 [1 << 17]byte + var x308 [1 << 17]byte + var x309 [1 << 17]byte + var x310 [1 << 17]byte + var x311 [1 << 17]byte + var x312 [1 << 17]byte + var x313 [1 << 17]byte + var x314 [1 << 17]byte + var x315 [1 << 17]byte + var x316 [1 << 17]byte + var x317 [1 << 17]byte + var x318 [1 << 17]byte + var x319 [1 << 17]byte + var x320 [1 << 17]byte + var x321 [1 << 17]byte + var x322 [1 << 17]byte + var x323 [1 << 17]byte + var x324 [1 << 17]byte + var x325 [1 << 17]byte + var x326 [1 << 17]byte + var x327 [1 << 17]byte + var x328 [1 << 17]byte + var x329 [1 << 17]byte + var x330 [1 << 17]byte + var x331 [1 << 17]byte + var x332 [1 << 17]byte + var x333 [1 << 17]byte + var x334 [1 << 17]byte + var x335 [1 << 17]byte + var x336 [1 << 17]byte + var x337 [1 << 17]byte + var x338 [1 << 17]byte + var x339 [1 << 17]byte + var x340 [1 << 17]byte + var x341 [1 << 17]byte + var x342 [1 << 17]byte + var x343 [1 << 17]byte + var x344 [1 << 17]byte + var x345 [1 << 17]byte + var x346 [1 << 17]byte + var x347 [1 << 17]byte + var x348 [1 << 17]byte + var x349 [1 << 17]byte + var x350 [1 << 17]byte + var x351 [1 << 17]byte + var x352 [1 << 17]byte + var x353 [1 << 17]byte + var x354 [1 << 17]byte + var x355 [1 << 17]byte + var x356 [1 << 17]byte + var x357 [1 << 17]byte + var x358 [1 << 17]byte + var x359 [1 << 17]byte + var x360 [1 << 17]byte + var x361 [1 << 17]byte + var x362 [1 << 17]byte + var x363 [1 << 17]byte + var x364 [1 << 17]byte + var x365 [1 << 17]byte + var x366 [1 << 17]byte + var x367 [1 << 17]byte + var x368 [1 << 17]byte + var x369 [1 << 17]byte + var x370 [1 << 17]byte + var x371 [1 << 17]byte + var x372 [1 << 17]byte + var x373 [1 << 17]byte + var x374 [1 << 17]byte + var x375 [1 << 17]byte + var x376 [1 << 17]byte + var x377 [1 << 17]byte + var x378 [1 << 17]byte + var x379 [1 << 17]byte + var x380 [1 << 17]byte + var x381 [1 << 17]byte + var x382 [1 << 17]byte + var x383 [1 << 17]byte + var x384 [1 << 17]byte + var x385 [1 << 17]byte + var x386 [1 << 17]byte + var x387 [1 << 17]byte + var x388 [1 << 17]byte + var x389 [1 << 17]byte + var x390 [1 << 17]byte + var x391 [1 << 17]byte + var x392 [1 << 17]byte + var x393 [1 << 17]byte + var x394 [1 << 17]byte + var x395 [1 << 17]byte + var x396 [1 << 17]byte + var x397 [1 << 17]byte + var x398 [1 << 17]byte + var x399 [1 << 17]byte + var x400 [1 << 17]byte + var x401 [1 << 17]byte + var x402 [1 << 17]byte + var x403 [1 << 17]byte + var x404 [1 << 17]byte + var x405 [1 << 17]byte + var x406 [1 << 17]byte + var x407 [1 << 17]byte + var x408 [1 << 17]byte + var x409 [1 << 17]byte + var x410 [1 << 17]byte + var x411 [1 << 17]byte + var x412 [1 << 17]byte + var x413 [1 << 17]byte + var x414 [1 << 17]byte + var x415 [1 << 17]byte + var x416 [1 << 17]byte + var x417 [1 << 17]byte + var x418 [1 << 17]byte + var x419 [1 << 17]byte + var x420 [1 << 17]byte + var x421 [1 << 17]byte + var x422 [1 << 17]byte + var x423 [1 << 17]byte + var x424 [1 << 17]byte + var x425 [1 << 17]byte + var x426 [1 << 17]byte + var x427 [1 << 17]byte + var x428 [1 << 17]byte + var x429 [1 << 17]byte + var x430 [1 << 17]byte + var x431 [1 << 17]byte + var x432 [1 << 17]byte + var x433 [1 << 17]byte + var x434 [1 << 17]byte + var x435 [1 << 17]byte + var x436 [1 << 17]byte + var x437 [1 << 17]byte + var x438 [1 << 17]byte + var x439 [1 << 17]byte + var x440 [1 << 17]byte + var x441 [1 << 17]byte + var x442 [1 << 17]byte + var x443 [1 << 17]byte + var x444 [1 << 17]byte + var x445 [1 << 17]byte + var x446 [1 << 17]byte + var x447 [1 << 17]byte + var x448 [1 << 17]byte + var x449 [1 << 17]byte + var x450 [1 << 17]byte + var x451 [1 << 17]byte + var x452 [1 << 17]byte + var x453 [1 << 17]byte + var x454 [1 << 17]byte + var x455 [1 << 17]byte + var x456 [1 << 17]byte + var x457 [1 << 17]byte + var x458 [1 << 17]byte + var x459 [1 << 17]byte + var x460 [1 << 17]byte + var x461 [1 << 17]byte + var x462 [1 << 17]byte + var x463 [1 << 17]byte + var x464 [1 << 17]byte + var x465 [1 << 17]byte + var x466 [1 << 17]byte + var x467 [1 << 17]byte + var x468 [1 << 17]byte + var x469 [1 << 17]byte + var x470 [1 << 17]byte + var x471 [1 << 17]byte + var x472 [1 << 17]byte + var x473 [1 << 17]byte + var x474 [1 << 17]byte + var x475 [1 << 17]byte + var x476 [1 << 17]byte + var x477 [1 << 17]byte + var x478 [1 << 17]byte + var x479 [1 << 17]byte + var x480 [1 << 17]byte + var x481 [1 << 17]byte + var x482 [1 << 17]byte + var x483 [1 << 17]byte + var x484 [1 << 17]byte + var x485 [1 << 17]byte + var x486 [1 << 17]byte + var x487 [1 << 17]byte + var x488 [1 << 17]byte + var x489 [1 << 17]byte + var x490 [1 << 17]byte + var x491 [1 << 17]byte + var x492 [1 << 17]byte + var x493 [1 << 17]byte + var x494 [1 << 17]byte + var x495 [1 << 17]byte + var x496 [1 << 17]byte + var x497 [1 << 17]byte + var x498 [1 << 17]byte + var x499 [1 << 17]byte + var x500 [1 << 17]byte + var x501 [1 << 17]byte + var x502 [1 << 17]byte + var x503 [1 << 17]byte + var x504 [1 << 17]byte + var x505 [1 << 17]byte + var x506 [1 << 17]byte + var x507 [1 << 17]byte + var x508 [1 << 17]byte + var x509 [1 << 17]byte + var x510 [1 << 17]byte + var x511 [1 << 17]byte + var x512 [1 << 17]byte + var x513 [1 << 17]byte + var x514 [1 << 17]byte + var x515 [1 << 17]byte + var x516 [1 << 17]byte + var x517 [1 << 17]byte + var x518 [1 << 17]byte + var x519 [1 << 17]byte + var x520 [1 << 17]byte + var x521 [1 << 17]byte + var x522 [1 << 17]byte + var x523 [1 << 17]byte + var x524 [1 << 17]byte + var x525 [1 << 17]byte + var x526 [1 << 17]byte + var x527 [1 << 17]byte + var x528 [1 << 17]byte + var x529 [1 << 17]byte + var x530 [1 << 17]byte + var x531 [1 << 17]byte + var x532 [1 << 17]byte + var x533 [1 << 17]byte + var x534 [1 << 17]byte + var x535 [1 << 17]byte + var x536 [1 << 17]byte + var x537 [1 << 17]byte + var x538 [1 << 17]byte + var x539 [1 << 17]byte + var x540 [1 << 17]byte + var x541 [1 << 17]byte + var x542 [1 << 17]byte + var x543 [1 << 17]byte + var x544 [1 << 17]byte + var x545 [1 << 17]byte + var x546 [1 << 17]byte + var x547 [1 << 17]byte + var x548 [1 << 17]byte + var x549 [1 << 17]byte + var x550 [1 << 17]byte + var x551 [1 << 17]byte + var x552 [1 << 17]byte + var x553 [1 << 17]byte + var x554 [1 << 17]byte + var x555 [1 << 17]byte + var x556 [1 << 17]byte + var x557 [1 << 17]byte + var x558 [1 << 17]byte + var x559 [1 << 17]byte + var x560 [1 << 17]byte + var x561 [1 << 17]byte + var x562 [1 << 17]byte + var x563 [1 << 17]byte + var x564 [1 << 17]byte + var x565 [1 << 17]byte + var x566 [1 << 17]byte + var x567 [1 << 17]byte + var x568 [1 << 17]byte + var x569 [1 << 17]byte + var x570 [1 << 17]byte + var x571 [1 << 17]byte + var x572 [1 << 17]byte + var x573 [1 << 17]byte + var x574 [1 << 17]byte + var x575 [1 << 17]byte + var x576 [1 << 17]byte + var x577 [1 << 17]byte + var x578 [1 << 17]byte + var x579 [1 << 17]byte + var x580 [1 << 17]byte + var x581 [1 << 17]byte + var x582 [1 << 17]byte + var x583 [1 << 17]byte + var x584 [1 << 17]byte + var x585 [1 << 17]byte + var x586 [1 << 17]byte + var x587 [1 << 17]byte + var x588 [1 << 17]byte + var x589 [1 << 17]byte + var x590 [1 << 17]byte + var x591 [1 << 17]byte + var x592 [1 << 17]byte + var x593 [1 << 17]byte + var x594 [1 << 17]byte + var x595 [1 << 17]byte + var x596 [1 << 17]byte + var x597 [1 << 17]byte + var x598 [1 << 17]byte + var x599 [1 << 17]byte + var x600 [1 << 17]byte + var x601 [1 << 17]byte + var x602 [1 << 17]byte + var x603 [1 << 17]byte + var x604 [1 << 17]byte + var x605 [1 << 17]byte + var x606 [1 << 17]byte + var x607 [1 << 17]byte + var x608 [1 << 17]byte + var x609 [1 << 17]byte + var x610 [1 << 17]byte + var x611 [1 << 17]byte + var x612 [1 << 17]byte + var x613 [1 << 17]byte + var x614 [1 << 17]byte + var x615 [1 << 17]byte + var x616 [1 << 17]byte + var x617 [1 << 17]byte + var x618 [1 << 17]byte + var x619 [1 << 17]byte + var x620 [1 << 17]byte + var x621 [1 << 17]byte + var x622 [1 << 17]byte + var x623 [1 << 17]byte + var x624 [1 << 17]byte + var x625 [1 << 17]byte + var x626 [1 << 17]byte + var x627 [1 << 17]byte + var x628 [1 << 17]byte + var x629 [1 << 17]byte + var x630 [1 << 17]byte + var x631 [1 << 17]byte + var x632 [1 << 17]byte + var x633 [1 << 17]byte + var x634 [1 << 17]byte + var x635 [1 << 17]byte + var x636 [1 << 17]byte + var x637 [1 << 17]byte + var x638 [1 << 17]byte + var x639 [1 << 17]byte + var x640 [1 << 17]byte + var x641 [1 << 17]byte + var x642 [1 << 17]byte + var x643 [1 << 17]byte + var x644 [1 << 17]byte + var x645 [1 << 17]byte + var x646 [1 << 17]byte + var x647 [1 << 17]byte + var x648 [1 << 17]byte + var x649 [1 << 17]byte + var x650 [1 << 17]byte + var x651 [1 << 17]byte + var x652 [1 << 17]byte + var x653 [1 << 17]byte + var x654 [1 << 17]byte + var x655 [1 << 17]byte + var x656 [1 << 17]byte + var x657 [1 << 17]byte + var x658 [1 << 17]byte + var x659 [1 << 17]byte + var x660 [1 << 17]byte + var x661 [1 << 17]byte + var x662 [1 << 17]byte + var x663 [1 << 17]byte + var x664 [1 << 17]byte + var x665 [1 << 17]byte + var x666 [1 << 17]byte + var x667 [1 << 17]byte + var x668 [1 << 17]byte + var x669 [1 << 17]byte + var x670 [1 << 17]byte + var x671 [1 << 17]byte + var x672 [1 << 17]byte + var x673 [1 << 17]byte + var x674 [1 << 17]byte + var x675 [1 << 17]byte + var x676 [1 << 17]byte + var x677 [1 << 17]byte + var x678 [1 << 17]byte + var x679 [1 << 17]byte + var x680 [1 << 17]byte + var x681 [1 << 17]byte + var x682 [1 << 17]byte + var x683 [1 << 17]byte + var x684 [1 << 17]byte + var x685 [1 << 17]byte + var x686 [1 << 17]byte + var x687 [1 << 17]byte + var x688 [1 << 17]byte + var x689 [1 << 17]byte + var x690 [1 << 17]byte + var x691 [1 << 17]byte + var x692 [1 << 17]byte + var x693 [1 << 17]byte + var x694 [1 << 17]byte + var x695 [1 << 17]byte + var x696 [1 << 17]byte + var x697 [1 << 17]byte + var x698 [1 << 17]byte + var x699 [1 << 17]byte + var x700 [1 << 17]byte + var x701 [1 << 17]byte + var x702 [1 << 17]byte + var x703 [1 << 17]byte + var x704 [1 << 17]byte + var x705 [1 << 17]byte + var x706 [1 << 17]byte + var x707 [1 << 17]byte + var x708 [1 << 17]byte + var x709 [1 << 17]byte + var x710 [1 << 17]byte + var x711 [1 << 17]byte + var x712 [1 << 17]byte + var x713 [1 << 17]byte + var x714 [1 << 17]byte + var x715 [1 << 17]byte + var x716 [1 << 17]byte + var x717 [1 << 17]byte + var x718 [1 << 17]byte + var x719 [1 << 17]byte + var x720 [1 << 17]byte + var x721 [1 << 17]byte + var x722 [1 << 17]byte + var x723 [1 << 17]byte + var x724 [1 << 17]byte + var x725 [1 << 17]byte + var x726 [1 << 17]byte + var x727 [1 << 17]byte + var x728 [1 << 17]byte + var x729 [1 << 17]byte + var x730 [1 << 17]byte + var x731 [1 << 17]byte + var x732 [1 << 17]byte + var x733 [1 << 17]byte + var x734 [1 << 17]byte + var x735 [1 << 17]byte + var x736 [1 << 17]byte + var x737 [1 << 17]byte + var x738 [1 << 17]byte + var x739 [1 << 17]byte + var x740 [1 << 17]byte + var x741 [1 << 17]byte + var x742 [1 << 17]byte + var x743 [1 << 17]byte + var x744 [1 << 17]byte + var x745 [1 << 17]byte + var x746 [1 << 17]byte + var x747 [1 << 17]byte + var x748 [1 << 17]byte + var x749 [1 << 17]byte + var x750 [1 << 17]byte + var x751 [1 << 17]byte + var x752 [1 << 17]byte + var x753 [1 << 17]byte + var x754 [1 << 17]byte + var x755 [1 << 17]byte + var x756 [1 << 17]byte + var x757 [1 << 17]byte + var x758 [1 << 17]byte + var x759 [1 << 17]byte + var x760 [1 << 17]byte + var x761 [1 << 17]byte + var x762 [1 << 17]byte + var x763 [1 << 17]byte + var x764 [1 << 17]byte + var x765 [1 << 17]byte + var x766 [1 << 17]byte + var x767 [1 << 17]byte + var x768 [1 << 17]byte + var x769 [1 << 17]byte + var x770 [1 << 17]byte + var x771 [1 << 17]byte + var x772 [1 << 17]byte + var x773 [1 << 17]byte + var x774 [1 << 17]byte + var x775 [1 << 17]byte + var x776 [1 << 17]byte + var x777 [1 << 17]byte + var x778 [1 << 17]byte + var x779 [1 << 17]byte + var x780 [1 << 17]byte + var x781 [1 << 17]byte + var x782 [1 << 17]byte + var x783 [1 << 17]byte + var x784 [1 << 17]byte + var x785 [1 << 17]byte + var x786 [1 << 17]byte + var x787 [1 << 17]byte + var x788 [1 << 17]byte + var x789 [1 << 17]byte + var x790 [1 << 17]byte + var x791 [1 << 17]byte + var x792 [1 << 17]byte + var x793 [1 << 17]byte + var x794 [1 << 17]byte + var x795 [1 << 17]byte + var x796 [1 << 17]byte + var x797 [1 << 17]byte + var x798 [1 << 17]byte + var x799 [1 << 17]byte + var x800 [1 << 17]byte + var x801 [1 << 17]byte + var x802 [1 << 17]byte + var x803 [1 << 17]byte + var x804 [1 << 17]byte + var x805 [1 << 17]byte + var x806 [1 << 17]byte + var x807 [1 << 17]byte + var x808 [1 << 17]byte + var x809 [1 << 17]byte + var x810 [1 << 17]byte + var x811 [1 << 17]byte + var x812 [1 << 17]byte + var x813 [1 << 17]byte + var x814 [1 << 17]byte + var x815 [1 << 17]byte + var x816 [1 << 17]byte + var x817 [1 << 17]byte + var x818 [1 << 17]byte + var x819 [1 << 17]byte + var x820 [1 << 17]byte + var x821 [1 << 17]byte + var x822 [1 << 17]byte + var x823 [1 << 17]byte + var x824 [1 << 17]byte + var x825 [1 << 17]byte + var x826 [1 << 17]byte + var x827 [1 << 17]byte + var x828 [1 << 17]byte + var x829 [1 << 17]byte + var x830 [1 << 17]byte + var x831 [1 << 17]byte + var x832 [1 << 17]byte + var x833 [1 << 17]byte + var x834 [1 << 17]byte + var x835 [1 << 17]byte + var x836 [1 << 17]byte + var x837 [1 << 17]byte + var x838 [1 << 17]byte + var x839 [1 << 17]byte + var x840 [1 << 17]byte + var x841 [1 << 17]byte + var x842 [1 << 17]byte + var x843 [1 << 17]byte + var x844 [1 << 17]byte + var x845 [1 << 17]byte + var x846 [1 << 17]byte + var x847 [1 << 17]byte + var x848 [1 << 17]byte + var x849 [1 << 17]byte + var x850 [1 << 17]byte + var x851 [1 << 17]byte + var x852 [1 << 17]byte + var x853 [1 << 17]byte + var x854 [1 << 17]byte + var x855 [1 << 17]byte + var x856 [1 << 17]byte + var x857 [1 << 17]byte + var x858 [1 << 17]byte + var x859 [1 << 17]byte + var x860 [1 << 17]byte + var x861 [1 << 17]byte + var x862 [1 << 17]byte + var x863 [1 << 17]byte + var x864 [1 << 17]byte + var x865 [1 << 17]byte + var x866 [1 << 17]byte + var x867 [1 << 17]byte + var x868 [1 << 17]byte + var x869 [1 << 17]byte + var x870 [1 << 17]byte + var x871 [1 << 17]byte + var x872 [1 << 17]byte + var x873 [1 << 17]byte + var x874 [1 << 17]byte + var x875 [1 << 17]byte + var x876 [1 << 17]byte + var x877 [1 << 17]byte + var x878 [1 << 17]byte + var x879 [1 << 17]byte + var x880 [1 << 17]byte + var x881 [1 << 17]byte + var x882 [1 << 17]byte + var x883 [1 << 17]byte + var x884 [1 << 17]byte + var x885 [1 << 17]byte + var x886 [1 << 17]byte + var x887 [1 << 17]byte + var x888 [1 << 17]byte + var x889 [1 << 17]byte + var x890 [1 << 17]byte + var x891 [1 << 17]byte + var x892 [1 << 17]byte + var x893 [1 << 17]byte + var x894 [1 << 17]byte + var x895 [1 << 17]byte + var x896 [1 << 17]byte + var x897 [1 << 17]byte + var x898 [1 << 17]byte + var x899 [1 << 17]byte + var x900 [1 << 17]byte + var x901 [1 << 17]byte + var x902 [1 << 17]byte + var x903 [1 << 17]byte + var x904 [1 << 17]byte + var x905 [1 << 17]byte + var x906 [1 << 17]byte + var x907 [1 << 17]byte + var x908 [1 << 17]byte + var x909 [1 << 17]byte + var x910 [1 << 17]byte + var x911 [1 << 17]byte + var x912 [1 << 17]byte + var x913 [1 << 17]byte + var x914 [1 << 17]byte + var x915 [1 << 17]byte + var x916 [1 << 17]byte + var x917 [1 << 17]byte + var x918 [1 << 17]byte + var x919 [1 << 17]byte + var x920 [1 << 17]byte + var x921 [1 << 17]byte + var x922 [1 << 17]byte + var x923 [1 << 17]byte + var x924 [1 << 17]byte + var x925 [1 << 17]byte + var x926 [1 << 17]byte + var x927 [1 << 17]byte + var x928 [1 << 17]byte + var x929 [1 << 17]byte + var x930 [1 << 17]byte + var x931 [1 << 17]byte + var x932 [1 << 17]byte + var x933 [1 << 17]byte + var x934 [1 << 17]byte + var x935 [1 << 17]byte + var x936 [1 << 17]byte + var x937 [1 << 17]byte + var x938 [1 << 17]byte + var x939 [1 << 17]byte + var x940 [1 << 17]byte + var x941 [1 << 17]byte + var x942 [1 << 17]byte + var x943 [1 << 17]byte + var x944 [1 << 17]byte + var x945 [1 << 17]byte + var x946 [1 << 17]byte + var x947 [1 << 17]byte + var x948 [1 << 17]byte + var x949 [1 << 17]byte + var x950 [1 << 17]byte + var x951 [1 << 17]byte + var x952 [1 << 17]byte + var x953 [1 << 17]byte + var x954 [1 << 17]byte + var x955 [1 << 17]byte + var x956 [1 << 17]byte + var x957 [1 << 17]byte + var x958 [1 << 17]byte + var x959 [1 << 17]byte + var x960 [1 << 17]byte + var x961 [1 << 17]byte + var x962 [1 << 17]byte + var x963 [1 << 17]byte + var x964 [1 << 17]byte + var x965 [1 << 17]byte + var x966 [1 << 17]byte + var x967 [1 << 17]byte + var x968 [1 << 17]byte + var x969 [1 << 17]byte + var x970 [1 << 17]byte + var x971 [1 << 17]byte + var x972 [1 << 17]byte + var x973 [1 << 17]byte + var x974 [1 << 17]byte + var x975 [1 << 17]byte + var x976 [1 << 17]byte + var x977 [1 << 17]byte + var x978 [1 << 17]byte + var x979 [1 << 17]byte + var x980 [1 << 17]byte + var x981 [1 << 17]byte + var x982 [1 << 17]byte + var x983 [1 << 17]byte + var x984 [1 << 17]byte + var x985 [1 << 17]byte + var x986 [1 << 17]byte + var x987 [1 << 17]byte + var x988 [1 << 17]byte + var x989 [1 << 17]byte + var x990 [1 << 17]byte + var x991 [1 << 17]byte + var x992 [1 << 17]byte + var x993 [1 << 17]byte + var x994 [1 << 17]byte + var x995 [1 << 17]byte + var x996 [1 << 17]byte + var x997 [1 << 17]byte + var x998 [1 << 17]byte + var x999 [1 << 17]byte + var x1000 [1 << 17]byte + var x1001 [1 << 17]byte + var x1002 [1 << 17]byte + var x1003 [1 << 17]byte + var x1004 [1 << 17]byte + var x1005 [1 << 17]byte + var x1006 [1 << 17]byte + var x1007 [1 << 17]byte + var x1008 [1 << 17]byte + var x1009 [1 << 17]byte + var x1010 [1 << 17]byte + var x1011 [1 << 17]byte + var x1012 [1 << 17]byte + var x1013 [1 << 17]byte + var x1014 [1 << 17]byte + var x1015 [1 << 17]byte + var x1016 [1 << 17]byte + var x1017 [1 << 17]byte + var x1018 [1 << 17]byte + var x1019 [1 << 17]byte + var x1020 [1 << 17]byte + var x1021 [1 << 17]byte + var x1022 [1 << 17]byte + var x1023 [1 << 17]byte + var x1024 [1 << 17]byte + var x1025 [1 << 17]byte + var x1026 [1 << 17]byte + var x1027 [1 << 17]byte + var x1028 [1 << 17]byte + var x1029 [1 << 17]byte + var x1030 [1 << 17]byte + var x1031 [1 << 17]byte + var x1032 [1 << 17]byte + var x1033 [1 << 17]byte + var x1034 [1 << 17]byte + var x1035 [1 << 17]byte + var x1036 [1 << 17]byte + var x1037 [1 << 17]byte + var x1038 [1 << 17]byte + var x1039 [1 << 17]byte + var x1040 [1 << 17]byte + var x1041 [1 << 17]byte + var x1042 [1 << 17]byte + var x1043 [1 << 17]byte + var x1044 [1 << 17]byte + var x1045 [1 << 17]byte + var x1046 [1 << 17]byte + var x1047 [1 << 17]byte + var x1048 [1 << 17]byte + var x1049 [1 << 17]byte + var x1050 [1 << 17]byte + var x1051 [1 << 17]byte + var x1052 [1 << 17]byte + var x1053 [1 << 17]byte + var x1054 [1 << 17]byte + var x1055 [1 << 17]byte + var x1056 [1 << 17]byte + var x1057 [1 << 17]byte + var x1058 [1 << 17]byte + var x1059 [1 << 17]byte + var x1060 [1 << 17]byte + var x1061 [1 << 17]byte + var x1062 [1 << 17]byte + var x1063 [1 << 17]byte + var x1064 [1 << 17]byte + var x1065 [1 << 17]byte + var x1066 [1 << 17]byte + var x1067 [1 << 17]byte + var x1068 [1 << 17]byte + var x1069 [1 << 17]byte + var x1070 [1 << 17]byte + var x1071 [1 << 17]byte + var x1072 [1 << 17]byte + var x1073 [1 << 17]byte + var x1074 [1 << 17]byte + var x1075 [1 << 17]byte + var x1076 [1 << 17]byte + var x1077 [1 << 17]byte + var x1078 [1 << 17]byte + var x1079 [1 << 17]byte + var x1080 [1 << 17]byte + var x1081 [1 << 17]byte + var x1082 [1 << 17]byte + var x1083 [1 << 17]byte + var x1084 [1 << 17]byte + var x1085 [1 << 17]byte + var x1086 [1 << 17]byte + var x1087 [1 << 17]byte + var x1088 [1 << 17]byte + var x1089 [1 << 17]byte + var x1090 [1 << 17]byte + var x1091 [1 << 17]byte + var x1092 [1 << 17]byte + var x1093 [1 << 17]byte + var x1094 [1 << 17]byte + var x1095 [1 << 17]byte + var x1096 [1 << 17]byte + var x1097 [1 << 17]byte + var x1098 [1 << 17]byte + var x1099 [1 << 17]byte + var x1100 [1 << 17]byte + var x1101 [1 << 17]byte + var x1102 [1 << 17]byte + var x1103 [1 << 17]byte + var x1104 [1 << 17]byte + var x1105 [1 << 17]byte + var x1106 [1 << 17]byte + var x1107 [1 << 17]byte + var x1108 [1 << 17]byte + var x1109 [1 << 17]byte + var x1110 [1 << 17]byte + var x1111 [1 << 17]byte + var x1112 [1 << 17]byte + var x1113 [1 << 17]byte + var x1114 [1 << 17]byte + var x1115 [1 << 17]byte + var x1116 [1 << 17]byte + var x1117 [1 << 17]byte + var x1118 [1 << 17]byte + var x1119 [1 << 17]byte + var x1120 [1 << 17]byte + var x1121 [1 << 17]byte + var x1122 [1 << 17]byte + var x1123 [1 << 17]byte + var x1124 [1 << 17]byte + var x1125 [1 << 17]byte + var x1126 [1 << 17]byte + var x1127 [1 << 17]byte + var x1128 [1 << 17]byte + var x1129 [1 << 17]byte + var x1130 [1 << 17]byte + var x1131 [1 << 17]byte + var x1132 [1 << 17]byte + var x1133 [1 << 17]byte + var x1134 [1 << 17]byte + var x1135 [1 << 17]byte + var x1136 [1 << 17]byte + var x1137 [1 << 17]byte + var x1138 [1 << 17]byte + var x1139 [1 << 17]byte + var x1140 [1 << 17]byte + var x1141 [1 << 17]byte + var x1142 [1 << 17]byte + var x1143 [1 << 17]byte + var x1144 [1 << 17]byte + var x1145 [1 << 17]byte + var x1146 [1 << 17]byte + var x1147 [1 << 17]byte + var x1148 [1 << 17]byte + var x1149 [1 << 17]byte + var x1150 [1 << 17]byte + var x1151 [1 << 17]byte + var x1152 [1 << 17]byte + var x1153 [1 << 17]byte + var x1154 [1 << 17]byte + var x1155 [1 << 17]byte + var x1156 [1 << 17]byte + var x1157 [1 << 17]byte + var x1158 [1 << 17]byte + var x1159 [1 << 17]byte + var x1160 [1 << 17]byte + var x1161 [1 << 17]byte + var x1162 [1 << 17]byte + var x1163 [1 << 17]byte + var x1164 [1 << 17]byte + var x1165 [1 << 17]byte + var x1166 [1 << 17]byte + var x1167 [1 << 17]byte + var x1168 [1 << 17]byte + var x1169 [1 << 17]byte + var x1170 [1 << 17]byte + var x1171 [1 << 17]byte + var x1172 [1 << 17]byte + var x1173 [1 << 17]byte + var x1174 [1 << 17]byte + var x1175 [1 << 17]byte + var x1176 [1 << 17]byte + var x1177 [1 << 17]byte + var x1178 [1 << 17]byte + var x1179 [1 << 17]byte + var x1180 [1 << 17]byte + var x1181 [1 << 17]byte + var x1182 [1 << 17]byte + var x1183 [1 << 17]byte + var x1184 [1 << 17]byte + var x1185 [1 << 17]byte + var x1186 [1 << 17]byte + var x1187 [1 << 17]byte + var x1188 [1 << 17]byte + var x1189 [1 << 17]byte + var x1190 [1 << 17]byte + var x1191 [1 << 17]byte + var x1192 [1 << 17]byte + var x1193 [1 << 17]byte + var x1194 [1 << 17]byte + var x1195 [1 << 17]byte + var x1196 [1 << 17]byte + var x1197 [1 << 17]byte + var x1198 [1 << 17]byte + var x1199 [1 << 17]byte + var x1200 [1 << 17]byte + var x1201 [1 << 17]byte + var x1202 [1 << 17]byte + var x1203 [1 << 17]byte + var x1204 [1 << 17]byte + var x1205 [1 << 17]byte + var x1206 [1 << 17]byte + var x1207 [1 << 17]byte + var x1208 [1 << 17]byte + var x1209 [1 << 17]byte + var x1210 [1 << 17]byte + var x1211 [1 << 17]byte + var x1212 [1 << 17]byte + var x1213 [1 << 17]byte + var x1214 [1 << 17]byte + var x1215 [1 << 17]byte + var x1216 [1 << 17]byte + var x1217 [1 << 17]byte + var x1218 [1 << 17]byte + var x1219 [1 << 17]byte + var x1220 [1 << 17]byte + var x1221 [1 << 17]byte + var x1222 [1 << 17]byte + var x1223 [1 << 17]byte + var x1224 [1 << 17]byte + var x1225 [1 << 17]byte + var x1226 [1 << 17]byte + var x1227 [1 << 17]byte + var x1228 [1 << 17]byte + var x1229 [1 << 17]byte + var x1230 [1 << 17]byte + var x1231 [1 << 17]byte + var x1232 [1 << 17]byte + var x1233 [1 << 17]byte + var x1234 [1 << 17]byte + var x1235 [1 << 17]byte + var x1236 [1 << 17]byte + var x1237 [1 << 17]byte + var x1238 [1 << 17]byte + var x1239 [1 << 17]byte + var x1240 [1 << 17]byte + var x1241 [1 << 17]byte + var x1242 [1 << 17]byte + var x1243 [1 << 17]byte + var x1244 [1 << 17]byte + var x1245 [1 << 17]byte + var x1246 [1 << 17]byte + var x1247 [1 << 17]byte + var x1248 [1 << 17]byte + var x1249 [1 << 17]byte + var x1250 [1 << 17]byte + var x1251 [1 << 17]byte + var x1252 [1 << 17]byte + var x1253 [1 << 17]byte + var x1254 [1 << 17]byte + var x1255 [1 << 17]byte + var x1256 [1 << 17]byte + var x1257 [1 << 17]byte + var x1258 [1 << 17]byte + var x1259 [1 << 17]byte + var x1260 [1 << 17]byte + var x1261 [1 << 17]byte + var x1262 [1 << 17]byte + var x1263 [1 << 17]byte + var x1264 [1 << 17]byte + var x1265 [1 << 17]byte + var x1266 [1 << 17]byte + var x1267 [1 << 17]byte + var x1268 [1 << 17]byte + var x1269 [1 << 17]byte + var x1270 [1 << 17]byte + var x1271 [1 << 17]byte + var x1272 [1 << 17]byte + var x1273 [1 << 17]byte + var x1274 [1 << 17]byte + var x1275 [1 << 17]byte + var x1276 [1 << 17]byte + var x1277 [1 << 17]byte + var x1278 [1 << 17]byte + var x1279 [1 << 17]byte + var x1280 [1 << 17]byte + var x1281 [1 << 17]byte + var x1282 [1 << 17]byte + var x1283 [1 << 17]byte + var x1284 [1 << 17]byte + var x1285 [1 << 17]byte + var x1286 [1 << 17]byte + var x1287 [1 << 17]byte + var x1288 [1 << 17]byte + var x1289 [1 << 17]byte + var x1290 [1 << 17]byte + var x1291 [1 << 17]byte + var x1292 [1 << 17]byte + var x1293 [1 << 17]byte + var x1294 [1 << 17]byte + var x1295 [1 << 17]byte + var x1296 [1 << 17]byte + var x1297 [1 << 17]byte + var x1298 [1 << 17]byte + var x1299 [1 << 17]byte + var x1300 [1 << 17]byte + var x1301 [1 << 17]byte + var x1302 [1 << 17]byte + var x1303 [1 << 17]byte + var x1304 [1 << 17]byte + var x1305 [1 << 17]byte + var x1306 [1 << 17]byte + var x1307 [1 << 17]byte + var x1308 [1 << 17]byte + var x1309 [1 << 17]byte + var x1310 [1 << 17]byte + var x1311 [1 << 17]byte + var x1312 [1 << 17]byte + var x1313 [1 << 17]byte + var x1314 [1 << 17]byte + var x1315 [1 << 17]byte + var x1316 [1 << 17]byte + var x1317 [1 << 17]byte + var x1318 [1 << 17]byte + var x1319 [1 << 17]byte + var x1320 [1 << 17]byte + var x1321 [1 << 17]byte + var x1322 [1 << 17]byte + var x1323 [1 << 17]byte + var x1324 [1 << 17]byte + var x1325 [1 << 17]byte + var x1326 [1 << 17]byte + var x1327 [1 << 17]byte + var x1328 [1 << 17]byte + var x1329 [1 << 17]byte + var x1330 [1 << 17]byte + var x1331 [1 << 17]byte + var x1332 [1 << 17]byte + var x1333 [1 << 17]byte + var x1334 [1 << 17]byte + var x1335 [1 << 17]byte + var x1336 [1 << 17]byte + var x1337 [1 << 17]byte + var x1338 [1 << 17]byte + var x1339 [1 << 17]byte + var x1340 [1 << 17]byte + var x1341 [1 << 17]byte + var x1342 [1 << 17]byte + var x1343 [1 << 17]byte + var x1344 [1 << 17]byte + var x1345 [1 << 17]byte + var x1346 [1 << 17]byte + var x1347 [1 << 17]byte + var x1348 [1 << 17]byte + var x1349 [1 << 17]byte + var x1350 [1 << 17]byte + var x1351 [1 << 17]byte + var x1352 [1 << 17]byte + var x1353 [1 << 17]byte + var x1354 [1 << 17]byte + var x1355 [1 << 17]byte + var x1356 [1 << 17]byte + var x1357 [1 << 17]byte + var x1358 [1 << 17]byte + var x1359 [1 << 17]byte + var x1360 [1 << 17]byte + var x1361 [1 << 17]byte + var x1362 [1 << 17]byte + var x1363 [1 << 17]byte + var x1364 [1 << 17]byte + var x1365 [1 << 17]byte + var x1366 [1 << 17]byte + var x1367 [1 << 17]byte + var x1368 [1 << 17]byte + var x1369 [1 << 17]byte + var x1370 [1 << 17]byte + var x1371 [1 << 17]byte + var x1372 [1 << 17]byte + var x1373 [1 << 17]byte + var x1374 [1 << 17]byte + var x1375 [1 << 17]byte + var x1376 [1 << 17]byte + var x1377 [1 << 17]byte + var x1378 [1 << 17]byte + var x1379 [1 << 17]byte + var x1380 [1 << 17]byte + var x1381 [1 << 17]byte + var x1382 [1 << 17]byte + var x1383 [1 << 17]byte + var x1384 [1 << 17]byte + var x1385 [1 << 17]byte + var x1386 [1 << 17]byte + var x1387 [1 << 17]byte + var x1388 [1 << 17]byte + var x1389 [1 << 17]byte + var x1390 [1 << 17]byte + var x1391 [1 << 17]byte + var x1392 [1 << 17]byte + var x1393 [1 << 17]byte + var x1394 [1 << 17]byte + var x1395 [1 << 17]byte + var x1396 [1 << 17]byte + var x1397 [1 << 17]byte + var x1398 [1 << 17]byte + var x1399 [1 << 17]byte + var x1400 [1 << 17]byte + var x1401 [1 << 17]byte + var x1402 [1 << 17]byte + var x1403 [1 << 17]byte + var x1404 [1 << 17]byte + var x1405 [1 << 17]byte + var x1406 [1 << 17]byte + var x1407 [1 << 17]byte + var x1408 [1 << 17]byte + var x1409 [1 << 17]byte + var x1410 [1 << 17]byte + var x1411 [1 << 17]byte + var x1412 [1 << 17]byte + var x1413 [1 << 17]byte + var x1414 [1 << 17]byte + var x1415 [1 << 17]byte + var x1416 [1 << 17]byte + var x1417 [1 << 17]byte + var x1418 [1 << 17]byte + var x1419 [1 << 17]byte + var x1420 [1 << 17]byte + var x1421 [1 << 17]byte + var x1422 [1 << 17]byte + var x1423 [1 << 17]byte + var x1424 [1 << 17]byte + var x1425 [1 << 17]byte + var x1426 [1 << 17]byte + var x1427 [1 << 17]byte + var x1428 [1 << 17]byte + var x1429 [1 << 17]byte + var x1430 [1 << 17]byte + var x1431 [1 << 17]byte + var x1432 [1 << 17]byte + var x1433 [1 << 17]byte + var x1434 [1 << 17]byte + var x1435 [1 << 17]byte + var x1436 [1 << 17]byte + var x1437 [1 << 17]byte + var x1438 [1 << 17]byte + var x1439 [1 << 17]byte + var x1440 [1 << 17]byte + var x1441 [1 << 17]byte + var x1442 [1 << 17]byte + var x1443 [1 << 17]byte + var x1444 [1 << 17]byte + var x1445 [1 << 17]byte + var x1446 [1 << 17]byte + var x1447 [1 << 17]byte + var x1448 [1 << 17]byte + var x1449 [1 << 17]byte + var x1450 [1 << 17]byte + var x1451 [1 << 17]byte + var x1452 [1 << 17]byte + var x1453 [1 << 17]byte + var x1454 [1 << 17]byte + var x1455 [1 << 17]byte + var x1456 [1 << 17]byte + var x1457 [1 << 17]byte + var x1458 [1 << 17]byte + var x1459 [1 << 17]byte + var x1460 [1 << 17]byte + var x1461 [1 << 17]byte + var x1462 [1 << 17]byte + var x1463 [1 << 17]byte + var x1464 [1 << 17]byte + var x1465 [1 << 17]byte + var x1466 [1 << 17]byte + var x1467 [1 << 17]byte + var x1468 [1 << 17]byte + var x1469 [1 << 17]byte + var x1470 [1 << 17]byte + var x1471 [1 << 17]byte + var x1472 [1 << 17]byte + var x1473 [1 << 17]byte + var x1474 [1 << 17]byte + var x1475 [1 << 17]byte + var x1476 [1 << 17]byte + var x1477 [1 << 17]byte + var x1478 [1 << 17]byte + var x1479 [1 << 17]byte + var x1480 [1 << 17]byte + var x1481 [1 << 17]byte + var x1482 [1 << 17]byte + var x1483 [1 << 17]byte + var x1484 [1 << 17]byte + var x1485 [1 << 17]byte + var x1486 [1 << 17]byte + var x1487 [1 << 17]byte + var x1488 [1 << 17]byte + var x1489 [1 << 17]byte + var x1490 [1 << 17]byte + var x1491 [1 << 17]byte + var x1492 [1 << 17]byte + var x1493 [1 << 17]byte + var x1494 [1 << 17]byte + var x1495 [1 << 17]byte + var x1496 [1 << 17]byte + var x1497 [1 << 17]byte + var x1498 [1 << 17]byte + var x1499 [1 << 17]byte + var x1500 [1 << 17]byte + var x1501 [1 << 17]byte + var x1502 [1 << 17]byte + var x1503 [1 << 17]byte + var x1504 [1 << 17]byte + var x1505 [1 << 17]byte + var x1506 [1 << 17]byte + var x1507 [1 << 17]byte + var x1508 [1 << 17]byte + var x1509 [1 << 17]byte + var x1510 [1 << 17]byte + var x1511 [1 << 17]byte + var x1512 [1 << 17]byte + var x1513 [1 << 17]byte + var x1514 [1 << 17]byte + var x1515 [1 << 17]byte + var x1516 [1 << 17]byte + var x1517 [1 << 17]byte + var x1518 [1 << 17]byte + var x1519 [1 << 17]byte + var x1520 [1 << 17]byte + var x1521 [1 << 17]byte + var x1522 [1 << 17]byte + var x1523 [1 << 17]byte + var x1524 [1 << 17]byte + var x1525 [1 << 17]byte + var x1526 [1 << 17]byte + var x1527 [1 << 17]byte + var x1528 [1 << 17]byte + var x1529 [1 << 17]byte + var x1530 [1 << 17]byte + var x1531 [1 << 17]byte + var x1532 [1 << 17]byte + var x1533 [1 << 17]byte + var x1534 [1 << 17]byte + var x1535 [1 << 17]byte + var x1536 [1 << 17]byte + var x1537 [1 << 17]byte + var x1538 [1 << 17]byte + var x1539 [1 << 17]byte + var x1540 [1 << 17]byte + var x1541 [1 << 17]byte + var x1542 [1 << 17]byte + var x1543 [1 << 17]byte + var x1544 [1 << 17]byte + var x1545 [1 << 17]byte + var x1546 [1 << 17]byte + var x1547 [1 << 17]byte + var x1548 [1 << 17]byte + var x1549 [1 << 17]byte + var x1550 [1 << 17]byte + var x1551 [1 << 17]byte + var x1552 [1 << 17]byte + var x1553 [1 << 17]byte + var x1554 [1 << 17]byte + var x1555 [1 << 17]byte + var x1556 [1 << 17]byte + var x1557 [1 << 17]byte + var x1558 [1 << 17]byte + var x1559 [1 << 17]byte + var x1560 [1 << 17]byte + var x1561 [1 << 17]byte + var x1562 [1 << 17]byte + var x1563 [1 << 17]byte + var x1564 [1 << 17]byte + var x1565 [1 << 17]byte + var x1566 [1 << 17]byte + var x1567 [1 << 17]byte + var x1568 [1 << 17]byte + var x1569 [1 << 17]byte + var x1570 [1 << 17]byte + var x1571 [1 << 17]byte + var x1572 [1 << 17]byte + var x1573 [1 << 17]byte + var x1574 [1 << 17]byte + var x1575 [1 << 17]byte + var x1576 [1 << 17]byte + var x1577 [1 << 17]byte + var x1578 [1 << 17]byte + var x1579 [1 << 17]byte + var x1580 [1 << 17]byte + var x1581 [1 << 17]byte + var x1582 [1 << 17]byte + var x1583 [1 << 17]byte + var x1584 [1 << 17]byte + var x1585 [1 << 17]byte + var x1586 [1 << 17]byte + var x1587 [1 << 17]byte + var x1588 [1 << 17]byte + var x1589 [1 << 17]byte + var x1590 [1 << 17]byte + var x1591 [1 << 17]byte + var x1592 [1 << 17]byte + var x1593 [1 << 17]byte + var x1594 [1 << 17]byte + var x1595 [1 << 17]byte + var x1596 [1 << 17]byte + var x1597 [1 << 17]byte + var x1598 [1 << 17]byte + var x1599 [1 << 17]byte + var x1600 [1 << 17]byte + var x1601 [1 << 17]byte + var x1602 [1 << 17]byte + var x1603 [1 << 17]byte + var x1604 [1 << 17]byte + var x1605 [1 << 17]byte + var x1606 [1 << 17]byte + var x1607 [1 << 17]byte + var x1608 [1 << 17]byte + var x1609 [1 << 17]byte + var x1610 [1 << 17]byte + var x1611 [1 << 17]byte + var x1612 [1 << 17]byte + var x1613 [1 << 17]byte + var x1614 [1 << 17]byte + var x1615 [1 << 17]byte + var x1616 [1 << 17]byte + var x1617 [1 << 17]byte + var x1618 [1 << 17]byte + var x1619 [1 << 17]byte + var x1620 [1 << 17]byte + var x1621 [1 << 17]byte + var x1622 [1 << 17]byte + var x1623 [1 << 17]byte + var x1624 [1 << 17]byte + var x1625 [1 << 17]byte + var x1626 [1 << 17]byte + var x1627 [1 << 17]byte + var x1628 [1 << 17]byte + var x1629 [1 << 17]byte + var x1630 [1 << 17]byte + var x1631 [1 << 17]byte + var x1632 [1 << 17]byte + var x1633 [1 << 17]byte + var x1634 [1 << 17]byte + var x1635 [1 << 17]byte + var x1636 [1 << 17]byte + var x1637 [1 << 17]byte + var x1638 [1 << 17]byte + var x1639 [1 << 17]byte + var x1640 [1 << 17]byte + var x1641 [1 << 17]byte + var x1642 [1 << 17]byte + var x1643 [1 << 17]byte + var x1644 [1 << 17]byte + var x1645 [1 << 17]byte + var x1646 [1 << 17]byte + var x1647 [1 << 17]byte + var x1648 [1 << 17]byte + var x1649 [1 << 17]byte + var x1650 [1 << 17]byte + var x1651 [1 << 17]byte + var x1652 [1 << 17]byte + var x1653 [1 << 17]byte + var x1654 [1 << 17]byte + var x1655 [1 << 17]byte + var x1656 [1 << 17]byte + var x1657 [1 << 17]byte + var x1658 [1 << 17]byte + var x1659 [1 << 17]byte + var x1660 [1 << 17]byte + var x1661 [1 << 17]byte + var x1662 [1 << 17]byte + var x1663 [1 << 17]byte + var x1664 [1 << 17]byte + var x1665 [1 << 17]byte + var x1666 [1 << 17]byte + var x1667 [1 << 17]byte + var x1668 [1 << 17]byte + var x1669 [1 << 17]byte + var x1670 [1 << 17]byte + var x1671 [1 << 17]byte + var x1672 [1 << 17]byte + var x1673 [1 << 17]byte + var x1674 [1 << 17]byte + var x1675 [1 << 17]byte + var x1676 [1 << 17]byte + var x1677 [1 << 17]byte + var x1678 [1 << 17]byte + var x1679 [1 << 17]byte + var x1680 [1 << 17]byte + var x1681 [1 << 17]byte + var x1682 [1 << 17]byte + var x1683 [1 << 17]byte + var x1684 [1 << 17]byte + var x1685 [1 << 17]byte + var x1686 [1 << 17]byte + var x1687 [1 << 17]byte + var x1688 [1 << 17]byte + var x1689 [1 << 17]byte + var x1690 [1 << 17]byte + var x1691 [1 << 17]byte + var x1692 [1 << 17]byte + var x1693 [1 << 17]byte + var x1694 [1 << 17]byte + var x1695 [1 << 17]byte + var x1696 [1 << 17]byte + var x1697 [1 << 17]byte + var x1698 [1 << 17]byte + var x1699 [1 << 17]byte + var x1700 [1 << 17]byte + var x1701 [1 << 17]byte + var x1702 [1 << 17]byte + var x1703 [1 << 17]byte + var x1704 [1 << 17]byte + var x1705 [1 << 17]byte + var x1706 [1 << 17]byte + var x1707 [1 << 17]byte + var x1708 [1 << 17]byte + var x1709 [1 << 17]byte + var x1710 [1 << 17]byte + var x1711 [1 << 17]byte + var x1712 [1 << 17]byte + var x1713 [1 << 17]byte + var x1714 [1 << 17]byte + var x1715 [1 << 17]byte + var x1716 [1 << 17]byte + var x1717 [1 << 17]byte + var x1718 [1 << 17]byte + var x1719 [1 << 17]byte + var x1720 [1 << 17]byte + var x1721 [1 << 17]byte + var x1722 [1 << 17]byte + var x1723 [1 << 17]byte + var x1724 [1 << 17]byte + var x1725 [1 << 17]byte + var x1726 [1 << 17]byte + var x1727 [1 << 17]byte + var x1728 [1 << 17]byte + var x1729 [1 << 17]byte + var x1730 [1 << 17]byte + var x1731 [1 << 17]byte + var x1732 [1 << 17]byte + var x1733 [1 << 17]byte + var x1734 [1 << 17]byte + var x1735 [1 << 17]byte + var x1736 [1 << 17]byte + var x1737 [1 << 17]byte + var x1738 [1 << 17]byte + var x1739 [1 << 17]byte + var x1740 [1 << 17]byte + var x1741 [1 << 17]byte + var x1742 [1 << 17]byte + var x1743 [1 << 17]byte + var x1744 [1 << 17]byte + var x1745 [1 << 17]byte + var x1746 [1 << 17]byte + var x1747 [1 << 17]byte + var x1748 [1 << 17]byte + var x1749 [1 << 17]byte + var x1750 [1 << 17]byte + var x1751 [1 << 17]byte + var x1752 [1 << 17]byte + var x1753 [1 << 17]byte + var x1754 [1 << 17]byte + var x1755 [1 << 17]byte + var x1756 [1 << 17]byte + var x1757 [1 << 17]byte + var x1758 [1 << 17]byte + var x1759 [1 << 17]byte + var x1760 [1 << 17]byte + var x1761 [1 << 17]byte + var x1762 [1 << 17]byte + var x1763 [1 << 17]byte + var x1764 [1 << 17]byte + var x1765 [1 << 17]byte + var x1766 [1 << 17]byte + var x1767 [1 << 17]byte + var x1768 [1 << 17]byte + var x1769 [1 << 17]byte + var x1770 [1 << 17]byte + var x1771 [1 << 17]byte + var x1772 [1 << 17]byte + var x1773 [1 << 17]byte + var x1774 [1 << 17]byte + var x1775 [1 << 17]byte + var x1776 [1 << 17]byte + var x1777 [1 << 17]byte + var x1778 [1 << 17]byte + var x1779 [1 << 17]byte + var x1780 [1 << 17]byte + var x1781 [1 << 17]byte + var x1782 [1 << 17]byte + var x1783 [1 << 17]byte + var x1784 [1 << 17]byte + var x1785 [1 << 17]byte + var x1786 [1 << 17]byte + var x1787 [1 << 17]byte + var x1788 [1 << 17]byte + var x1789 [1 << 17]byte + var x1790 [1 << 17]byte + var x1791 [1 << 17]byte + var x1792 [1 << 17]byte + var x1793 [1 << 17]byte + var x1794 [1 << 17]byte + var x1795 [1 << 17]byte + var x1796 [1 << 17]byte + var x1797 [1 << 17]byte + var x1798 [1 << 17]byte + var x1799 [1 << 17]byte + var x1800 [1 << 17]byte + var x1801 [1 << 17]byte + var x1802 [1 << 17]byte + var x1803 [1 << 17]byte + var x1804 [1 << 17]byte + var x1805 [1 << 17]byte + var x1806 [1 << 17]byte + var x1807 [1 << 17]byte + var x1808 [1 << 17]byte + var x1809 [1 << 17]byte + var x1810 [1 << 17]byte + var x1811 [1 << 17]byte + var x1812 [1 << 17]byte + var x1813 [1 << 17]byte + var x1814 [1 << 17]byte + var x1815 [1 << 17]byte + var x1816 [1 << 17]byte + var x1817 [1 << 17]byte + var x1818 [1 << 17]byte + var x1819 [1 << 17]byte + var x1820 [1 << 17]byte + var x1821 [1 << 17]byte + var x1822 [1 << 17]byte + var x1823 [1 << 17]byte + var x1824 [1 << 17]byte + var x1825 [1 << 17]byte + var x1826 [1 << 17]byte + var x1827 [1 << 17]byte + var x1828 [1 << 17]byte + var x1829 [1 << 17]byte + var x1830 [1 << 17]byte + var x1831 [1 << 17]byte + var x1832 [1 << 17]byte + var x1833 [1 << 17]byte + var x1834 [1 << 17]byte + var x1835 [1 << 17]byte + var x1836 [1 << 17]byte + var x1837 [1 << 17]byte + var x1838 [1 << 17]byte + var x1839 [1 << 17]byte + var x1840 [1 << 17]byte + var x1841 [1 << 17]byte + var x1842 [1 << 17]byte + var x1843 [1 << 17]byte + var x1844 [1 << 17]byte + var x1845 [1 << 17]byte + var x1846 [1 << 17]byte + var x1847 [1 << 17]byte + var x1848 [1 << 17]byte + var x1849 [1 << 17]byte + var x1850 [1 << 17]byte + var x1851 [1 << 17]byte + var x1852 [1 << 17]byte + var x1853 [1 << 17]byte + var x1854 [1 << 17]byte + var x1855 [1 << 17]byte + var x1856 [1 << 17]byte + var x1857 [1 << 17]byte + var x1858 [1 << 17]byte + var x1859 [1 << 17]byte + var x1860 [1 << 17]byte + var x1861 [1 << 17]byte + var x1862 [1 << 17]byte + var x1863 [1 << 17]byte + var x1864 [1 << 17]byte + var x1865 [1 << 17]byte + var x1866 [1 << 17]byte + var x1867 [1 << 17]byte + var x1868 [1 << 17]byte + var x1869 [1 << 17]byte + var x1870 [1 << 17]byte + var x1871 [1 << 17]byte + var x1872 [1 << 17]byte + var x1873 [1 << 17]byte + var x1874 [1 << 17]byte + var x1875 [1 << 17]byte + var x1876 [1 << 17]byte + var x1877 [1 << 17]byte + var x1878 [1 << 17]byte + var x1879 [1 << 17]byte + var x1880 [1 << 17]byte + var x1881 [1 << 17]byte + var x1882 [1 << 17]byte + var x1883 [1 << 17]byte + var x1884 [1 << 17]byte + var x1885 [1 << 17]byte + var x1886 [1 << 17]byte + var x1887 [1 << 17]byte + var x1888 [1 << 17]byte + var x1889 [1 << 17]byte + var x1890 [1 << 17]byte + var x1891 [1 << 17]byte + var x1892 [1 << 17]byte + var x1893 [1 << 17]byte + var x1894 [1 << 17]byte + var x1895 [1 << 17]byte + var x1896 [1 << 17]byte + var x1897 [1 << 17]byte + var x1898 [1 << 17]byte + var x1899 [1 << 17]byte + var x1900 [1 << 17]byte + var x1901 [1 << 17]byte + var x1902 [1 << 17]byte + var x1903 [1 << 17]byte + var x1904 [1 << 17]byte + var x1905 [1 << 17]byte + var x1906 [1 << 17]byte + var x1907 [1 << 17]byte + var x1908 [1 << 17]byte + var x1909 [1 << 17]byte + var x1910 [1 << 17]byte + var x1911 [1 << 17]byte + var x1912 [1 << 17]byte + var x1913 [1 << 17]byte + var x1914 [1 << 17]byte + var x1915 [1 << 17]byte + var x1916 [1 << 17]byte + var x1917 [1 << 17]byte + var x1918 [1 << 17]byte + var x1919 [1 << 17]byte + var x1920 [1 << 17]byte + var x1921 [1 << 17]byte + var x1922 [1 << 17]byte + var x1923 [1 << 17]byte + var x1924 [1 << 17]byte + var x1925 [1 << 17]byte + var x1926 [1 << 17]byte + var x1927 [1 << 17]byte + var x1928 [1 << 17]byte + var x1929 [1 << 17]byte + var x1930 [1 << 17]byte + var x1931 [1 << 17]byte + var x1932 [1 << 17]byte + var x1933 [1 << 17]byte + var x1934 [1 << 17]byte + var x1935 [1 << 17]byte + var x1936 [1 << 17]byte + var x1937 [1 << 17]byte + var x1938 [1 << 17]byte + var x1939 [1 << 17]byte + var x1940 [1 << 17]byte + var x1941 [1 << 17]byte + var x1942 [1 << 17]byte + var x1943 [1 << 17]byte + var x1944 [1 << 17]byte + var x1945 [1 << 17]byte + var x1946 [1 << 17]byte + var x1947 [1 << 17]byte + var x1948 [1 << 17]byte + var x1949 [1 << 17]byte + var x1950 [1 << 17]byte + var x1951 [1 << 17]byte + var x1952 [1 << 17]byte + var x1953 [1 << 17]byte + var x1954 [1 << 17]byte + var x1955 [1 << 17]byte + var x1956 [1 << 17]byte + var x1957 [1 << 17]byte + var x1958 [1 << 17]byte + var x1959 [1 << 17]byte + var x1960 [1 << 17]byte + var x1961 [1 << 17]byte + var x1962 [1 << 17]byte + var x1963 [1 << 17]byte + var x1964 [1 << 17]byte + var x1965 [1 << 17]byte + var x1966 [1 << 17]byte + var x1967 [1 << 17]byte + var x1968 [1 << 17]byte + var x1969 [1 << 17]byte + var x1970 [1 << 17]byte + var x1971 [1 << 17]byte + var x1972 [1 << 17]byte + var x1973 [1 << 17]byte + var x1974 [1 << 17]byte + var x1975 [1 << 17]byte + var x1976 [1 << 17]byte + var x1977 [1 << 17]byte + var x1978 [1 << 17]byte + var x1979 [1 << 17]byte + var x1980 [1 << 17]byte + var x1981 [1 << 17]byte + var x1982 [1 << 17]byte + var x1983 [1 << 17]byte + var x1984 [1 << 17]byte + var x1985 [1 << 17]byte + var x1986 [1 << 17]byte + var x1987 [1 << 17]byte + var x1988 [1 << 17]byte + var x1989 [1 << 17]byte + var x1990 [1 << 17]byte + var x1991 [1 << 17]byte + var x1992 [1 << 17]byte + var x1993 [1 << 17]byte + var x1994 [1 << 17]byte + var x1995 [1 << 17]byte + var x1996 [1 << 17]byte + var x1997 [1 << 17]byte + var x1998 [1 << 17]byte + var x1999 [1 << 17]byte + var x2000 [1 << 17]byte + var x2001 [1 << 17]byte + var x2002 [1 << 17]byte + var x2003 [1 << 17]byte + var x2004 [1 << 17]byte + var x2005 [1 << 17]byte + var x2006 [1 << 17]byte + var x2007 [1 << 17]byte + var x2008 [1 << 17]byte + var x2009 [1 << 17]byte + var x2010 [1 << 17]byte + var x2011 [1 << 17]byte + var x2012 [1 << 17]byte + var x2013 [1 << 17]byte + var x2014 [1 << 17]byte + var x2015 [1 << 17]byte + var x2016 [1 << 17]byte + var x2017 [1 << 17]byte + var x2018 [1 << 17]byte + var x2019 [1 << 17]byte + var x2020 [1 << 17]byte + var x2021 [1 << 17]byte + var x2022 [1 << 17]byte + var x2023 [1 << 17]byte + var x2024 [1 << 17]byte + var x2025 [1 << 17]byte + var x2026 [1 << 17]byte + var x2027 [1 << 17]byte + var x2028 [1 << 17]byte + var x2029 [1 << 17]byte + var x2030 [1 << 17]byte + var x2031 [1 << 17]byte + var x2032 [1 << 17]byte + var x2033 [1 << 17]byte + var x2034 [1 << 17]byte + var x2035 [1 << 17]byte + var x2036 [1 << 17]byte + var x2037 [1 << 17]byte + var x2038 [1 << 17]byte + var x2039 [1 << 17]byte + var x2040 [1 << 17]byte + var x2041 [1 << 17]byte + var x2042 [1 << 17]byte + var x2043 [1 << 17]byte + var x2044 [1 << 17]byte + var x2045 [1 << 17]byte + var x2046 [1 << 17]byte + var x2047 [1 << 17]byte + var x2048 [1 << 17]byte + var x2049 [1 << 17]byte + var x2050 [1 << 17]byte + var x2051 [1 << 17]byte + var x2052 [1 << 17]byte + var x2053 [1 << 17]byte + var x2054 [1 << 17]byte + var x2055 [1 << 17]byte + var x2056 [1 << 17]byte + var x2057 [1 << 17]byte + var x2058 [1 << 17]byte + var x2059 [1 << 17]byte + var x2060 [1 << 17]byte + var x2061 [1 << 17]byte + var x2062 [1 << 17]byte + var x2063 [1 << 17]byte + var x2064 [1 << 17]byte + var x2065 [1 << 17]byte + var x2066 [1 << 17]byte + var x2067 [1 << 17]byte + var x2068 [1 << 17]byte + var x2069 [1 << 17]byte + var x2070 [1 << 17]byte + var x2071 [1 << 17]byte + var x2072 [1 << 17]byte + var x2073 [1 << 17]byte + var x2074 [1 << 17]byte + var x2075 [1 << 17]byte + var x2076 [1 << 17]byte + var x2077 [1 << 17]byte + var x2078 [1 << 17]byte + var x2079 [1 << 17]byte + var x2080 [1 << 17]byte + var x2081 [1 << 17]byte + var x2082 [1 << 17]byte + var x2083 [1 << 17]byte + var x2084 [1 << 17]byte + var x2085 [1 << 17]byte + var x2086 [1 << 17]byte + var x2087 [1 << 17]byte + var x2088 [1 << 17]byte + var x2089 [1 << 17]byte + var x2090 [1 << 17]byte + var x2091 [1 << 17]byte + var x2092 [1 << 17]byte + var x2093 [1 << 17]byte + var x2094 [1 << 17]byte + var x2095 [1 << 17]byte + var x2096 [1 << 17]byte + var x2097 [1 << 17]byte + var x2098 [1 << 17]byte + var x2099 [1 << 17]byte + var x2100 [1 << 17]byte + var x2101 [1 << 17]byte + var x2102 [1 << 17]byte + var x2103 [1 << 17]byte + var x2104 [1 << 17]byte + var x2105 [1 << 17]byte + var x2106 [1 << 17]byte + var x2107 [1 << 17]byte + var x2108 [1 << 17]byte + var x2109 [1 << 17]byte + var x2110 [1 << 17]byte + var x2111 [1 << 17]byte + var x2112 [1 << 17]byte + var x2113 [1 << 17]byte + var x2114 [1 << 17]byte + var x2115 [1 << 17]byte + var x2116 [1 << 17]byte + var x2117 [1 << 17]byte + var x2118 [1 << 17]byte + var x2119 [1 << 17]byte + var x2120 [1 << 17]byte + var x2121 [1 << 17]byte + var x2122 [1 << 17]byte + var x2123 [1 << 17]byte + var x2124 [1 << 17]byte + var x2125 [1 << 17]byte + var x2126 [1 << 17]byte + var x2127 [1 << 17]byte + var x2128 [1 << 17]byte + var x2129 [1 << 17]byte + var x2130 [1 << 17]byte + var x2131 [1 << 17]byte + var x2132 [1 << 17]byte + var x2133 [1 << 17]byte + var x2134 [1 << 17]byte + var x2135 [1 << 17]byte + var x2136 [1 << 17]byte + var x2137 [1 << 17]byte + var x2138 [1 << 17]byte + var x2139 [1 << 17]byte + var x2140 [1 << 17]byte + var x2141 [1 << 17]byte + var x2142 [1 << 17]byte + var x2143 [1 << 17]byte + var x2144 [1 << 17]byte + var x2145 [1 << 17]byte + var x2146 [1 << 17]byte + var x2147 [1 << 17]byte + var x2148 [1 << 17]byte + var x2149 [1 << 17]byte + var x2150 [1 << 17]byte + var x2151 [1 << 17]byte + var x2152 [1 << 17]byte + var x2153 [1 << 17]byte + var x2154 [1 << 17]byte + var x2155 [1 << 17]byte + var x2156 [1 << 17]byte + var x2157 [1 << 17]byte + var x2158 [1 << 17]byte + var x2159 [1 << 17]byte + var x2160 [1 << 17]byte + var x2161 [1 << 17]byte + var x2162 [1 << 17]byte + var x2163 [1 << 17]byte + var x2164 [1 << 17]byte + var x2165 [1 << 17]byte + var x2166 [1 << 17]byte + var x2167 [1 << 17]byte + var x2168 [1 << 17]byte + var x2169 [1 << 17]byte + var x2170 [1 << 17]byte + var x2171 [1 << 17]byte + var x2172 [1 << 17]byte + var x2173 [1 << 17]byte + var x2174 [1 << 17]byte + var x2175 [1 << 17]byte + var x2176 [1 << 17]byte + var x2177 [1 << 17]byte + var x2178 [1 << 17]byte + var x2179 [1 << 17]byte + var x2180 [1 << 17]byte + var x2181 [1 << 17]byte + var x2182 [1 << 17]byte + var x2183 [1 << 17]byte + var x2184 [1 << 17]byte + var x2185 [1 << 17]byte + var x2186 [1 << 17]byte + var x2187 [1 << 17]byte + var x2188 [1 << 17]byte + var x2189 [1 << 17]byte + var x2190 [1 << 17]byte + var x2191 [1 << 17]byte + var x2192 [1 << 17]byte + var x2193 [1 << 17]byte + var x2194 [1 << 17]byte + var x2195 [1 << 17]byte + var x2196 [1 << 17]byte + var x2197 [1 << 17]byte + var x2198 [1 << 17]byte + var x2199 [1 << 17]byte + var x2200 [1 << 17]byte + var x2201 [1 << 17]byte + var x2202 [1 << 17]byte + var x2203 [1 << 17]byte + var x2204 [1 << 17]byte + var x2205 [1 << 17]byte + var x2206 [1 << 17]byte + var x2207 [1 << 17]byte + var x2208 [1 << 17]byte + var x2209 [1 << 17]byte + var x2210 [1 << 17]byte + var x2211 [1 << 17]byte + var x2212 [1 << 17]byte + var x2213 [1 << 17]byte + var x2214 [1 << 17]byte + var x2215 [1 << 17]byte + var x2216 [1 << 17]byte + var x2217 [1 << 17]byte + var x2218 [1 << 17]byte + var x2219 [1 << 17]byte + var x2220 [1 << 17]byte + var x2221 [1 << 17]byte + var x2222 [1 << 17]byte + var x2223 [1 << 17]byte + var x2224 [1 << 17]byte + var x2225 [1 << 17]byte + var x2226 [1 << 17]byte + var x2227 [1 << 17]byte + var x2228 [1 << 17]byte + var x2229 [1 << 17]byte + var x2230 [1 << 17]byte + var x2231 [1 << 17]byte + var x2232 [1 << 17]byte + var x2233 [1 << 17]byte + var x2234 [1 << 17]byte + var x2235 [1 << 17]byte + var x2236 [1 << 17]byte + var x2237 [1 << 17]byte + var x2238 [1 << 17]byte + var x2239 [1 << 17]byte + var x2240 [1 << 17]byte + var x2241 [1 << 17]byte + var x2242 [1 << 17]byte + var x2243 [1 << 17]byte + var x2244 [1 << 17]byte + var x2245 [1 << 17]byte + var x2246 [1 << 17]byte + var x2247 [1 << 17]byte + var x2248 [1 << 17]byte + var x2249 [1 << 17]byte + var x2250 [1 << 17]byte + var x2251 [1 << 17]byte + var x2252 [1 << 17]byte + var x2253 [1 << 17]byte + var x2254 [1 << 17]byte + var x2255 [1 << 17]byte + var x2256 [1 << 17]byte + var x2257 [1 << 17]byte + var x2258 [1 << 17]byte + var x2259 [1 << 17]byte + var x2260 [1 << 17]byte + var x2261 [1 << 17]byte + var x2262 [1 << 17]byte + var x2263 [1 << 17]byte + var x2264 [1 << 17]byte + var x2265 [1 << 17]byte + var x2266 [1 << 17]byte + var x2267 [1 << 17]byte + var x2268 [1 << 17]byte + var x2269 [1 << 17]byte + var x2270 [1 << 17]byte + var x2271 [1 << 17]byte + var x2272 [1 << 17]byte + var x2273 [1 << 17]byte + var x2274 [1 << 17]byte + var x2275 [1 << 17]byte + var x2276 [1 << 17]byte + var x2277 [1 << 17]byte + var x2278 [1 << 17]byte + var x2279 [1 << 17]byte + var x2280 [1 << 17]byte + var x2281 [1 << 17]byte + var x2282 [1 << 17]byte + var x2283 [1 << 17]byte + var x2284 [1 << 17]byte + var x2285 [1 << 17]byte + var x2286 [1 << 17]byte + var x2287 [1 << 17]byte + var x2288 [1 << 17]byte + var x2289 [1 << 17]byte + var x2290 [1 << 17]byte + var x2291 [1 << 17]byte + var x2292 [1 << 17]byte + var x2293 [1 << 17]byte + var x2294 [1 << 17]byte + var x2295 [1 << 17]byte + var x2296 [1 << 17]byte + var x2297 [1 << 17]byte + var x2298 [1 << 17]byte + var x2299 [1 << 17]byte + var x2300 [1 << 17]byte + var x2301 [1 << 17]byte + var x2302 [1 << 17]byte + var x2303 [1 << 17]byte + var x2304 [1 << 17]byte + var x2305 [1 << 17]byte + var x2306 [1 << 17]byte + var x2307 [1 << 17]byte + var x2308 [1 << 17]byte + var x2309 [1 << 17]byte + var x2310 [1 << 17]byte + var x2311 [1 << 17]byte + var x2312 [1 << 17]byte + var x2313 [1 << 17]byte + var x2314 [1 << 17]byte + var x2315 [1 << 17]byte + var x2316 [1 << 17]byte + var x2317 [1 << 17]byte + var x2318 [1 << 17]byte + var x2319 [1 << 17]byte + var x2320 [1 << 17]byte + var x2321 [1 << 17]byte + var x2322 [1 << 17]byte + var x2323 [1 << 17]byte + var x2324 [1 << 17]byte + var x2325 [1 << 17]byte + var x2326 [1 << 17]byte + var x2327 [1 << 17]byte + var x2328 [1 << 17]byte + var x2329 [1 << 17]byte + var x2330 [1 << 17]byte + var x2331 [1 << 17]byte + var x2332 [1 << 17]byte + var x2333 [1 << 17]byte + var x2334 [1 << 17]byte + var x2335 [1 << 17]byte + var x2336 [1 << 17]byte + var x2337 [1 << 17]byte + var x2338 [1 << 17]byte + var x2339 [1 << 17]byte + var x2340 [1 << 17]byte + var x2341 [1 << 17]byte + var x2342 [1 << 17]byte + var x2343 [1 << 17]byte + var x2344 [1 << 17]byte + var x2345 [1 << 17]byte + var x2346 [1 << 17]byte + var x2347 [1 << 17]byte + var x2348 [1 << 17]byte + var x2349 [1 << 17]byte + var x2350 [1 << 17]byte + var x2351 [1 << 17]byte + var x2352 [1 << 17]byte + var x2353 [1 << 17]byte + var x2354 [1 << 17]byte + var x2355 [1 << 17]byte + var x2356 [1 << 17]byte + var x2357 [1 << 17]byte + var x2358 [1 << 17]byte + var x2359 [1 << 17]byte + var x2360 [1 << 17]byte + var x2361 [1 << 17]byte + var x2362 [1 << 17]byte + var x2363 [1 << 17]byte + var x2364 [1 << 17]byte + var x2365 [1 << 17]byte + var x2366 [1 << 17]byte + var x2367 [1 << 17]byte + var x2368 [1 << 17]byte + var x2369 [1 << 17]byte + var x2370 [1 << 17]byte + var x2371 [1 << 17]byte + var x2372 [1 << 17]byte + var x2373 [1 << 17]byte + var x2374 [1 << 17]byte + var x2375 [1 << 17]byte + var x2376 [1 << 17]byte + var x2377 [1 << 17]byte + var x2378 [1 << 17]byte + var x2379 [1 << 17]byte + var x2380 [1 << 17]byte + var x2381 [1 << 17]byte + var x2382 [1 << 17]byte + var x2383 [1 << 17]byte + var x2384 [1 << 17]byte + var x2385 [1 << 17]byte + var x2386 [1 << 17]byte + var x2387 [1 << 17]byte + var x2388 [1 << 17]byte + var x2389 [1 << 17]byte + var x2390 [1 << 17]byte + var x2391 [1 << 17]byte + var x2392 [1 << 17]byte + var x2393 [1 << 17]byte + var x2394 [1 << 17]byte + var x2395 [1 << 17]byte + var x2396 [1 << 17]byte + var x2397 [1 << 17]byte + var x2398 [1 << 17]byte + var x2399 [1 << 17]byte + var x2400 [1 << 17]byte + var x2401 [1 << 17]byte + var x2402 [1 << 17]byte + var x2403 [1 << 17]byte + var x2404 [1 << 17]byte + var x2405 [1 << 17]byte + var x2406 [1 << 17]byte + var x2407 [1 << 17]byte + var x2408 [1 << 17]byte + var x2409 [1 << 17]byte + var x2410 [1 << 17]byte + var x2411 [1 << 17]byte + var x2412 [1 << 17]byte + var x2413 [1 << 17]byte + var x2414 [1 << 17]byte + var x2415 [1 << 17]byte + var x2416 [1 << 17]byte + var x2417 [1 << 17]byte + var x2418 [1 << 17]byte + var x2419 [1 << 17]byte + var x2420 [1 << 17]byte + var x2421 [1 << 17]byte + var x2422 [1 << 17]byte + var x2423 [1 << 17]byte + var x2424 [1 << 17]byte + var x2425 [1 << 17]byte + var x2426 [1 << 17]byte + var x2427 [1 << 17]byte + var x2428 [1 << 17]byte + var x2429 [1 << 17]byte + var x2430 [1 << 17]byte + var x2431 [1 << 17]byte + var x2432 [1 << 17]byte + var x2433 [1 << 17]byte + var x2434 [1 << 17]byte + var x2435 [1 << 17]byte + var x2436 [1 << 17]byte + var x2437 [1 << 17]byte + var x2438 [1 << 17]byte + var x2439 [1 << 17]byte + var x2440 [1 << 17]byte + var x2441 [1 << 17]byte + var x2442 [1 << 17]byte + var x2443 [1 << 17]byte + var x2444 [1 << 17]byte + var x2445 [1 << 17]byte + var x2446 [1 << 17]byte + var x2447 [1 << 17]byte + var x2448 [1 << 17]byte + var x2449 [1 << 17]byte + var x2450 [1 << 17]byte + var x2451 [1 << 17]byte + var x2452 [1 << 17]byte + var x2453 [1 << 17]byte + var x2454 [1 << 17]byte + var x2455 [1 << 17]byte + var x2456 [1 << 17]byte + var x2457 [1 << 17]byte + var x2458 [1 << 17]byte + var x2459 [1 << 17]byte + var x2460 [1 << 17]byte + var x2461 [1 << 17]byte + var x2462 [1 << 17]byte + var x2463 [1 << 17]byte + var x2464 [1 << 17]byte + var x2465 [1 << 17]byte + var x2466 [1 << 17]byte + var x2467 [1 << 17]byte + var x2468 [1 << 17]byte + var x2469 [1 << 17]byte + var x2470 [1 << 17]byte + var x2471 [1 << 17]byte + var x2472 [1 << 17]byte + var x2473 [1 << 17]byte + var x2474 [1 << 17]byte + var x2475 [1 << 17]byte + var x2476 [1 << 17]byte + var x2477 [1 << 17]byte + var x2478 [1 << 17]byte + var x2479 [1 << 17]byte + var x2480 [1 << 17]byte + var x2481 [1 << 17]byte + var x2482 [1 << 17]byte + var x2483 [1 << 17]byte + var x2484 [1 << 17]byte + var x2485 [1 << 17]byte + var x2486 [1 << 17]byte + var x2487 [1 << 17]byte + var x2488 [1 << 17]byte + var x2489 [1 << 17]byte + var x2490 [1 << 17]byte + var x2491 [1 << 17]byte + var x2492 [1 << 17]byte + var x2493 [1 << 17]byte + var x2494 [1 << 17]byte + var x2495 [1 << 17]byte + var x2496 [1 << 17]byte + var x2497 [1 << 17]byte + var x2498 [1 << 17]byte + var x2499 [1 << 17]byte + var x2500 [1 << 17]byte + var x2501 [1 << 17]byte + var x2502 [1 << 17]byte + var x2503 [1 << 17]byte + var x2504 [1 << 17]byte + var x2505 [1 << 17]byte + var x2506 [1 << 17]byte + var x2507 [1 << 17]byte + var x2508 [1 << 17]byte + var x2509 [1 << 17]byte + var x2510 [1 << 17]byte + var x2511 [1 << 17]byte + var x2512 [1 << 17]byte + var x2513 [1 << 17]byte + var x2514 [1 << 17]byte + var x2515 [1 << 17]byte + var x2516 [1 << 17]byte + var x2517 [1 << 17]byte + var x2518 [1 << 17]byte + var x2519 [1 << 17]byte + var x2520 [1 << 17]byte + var x2521 [1 << 17]byte + var x2522 [1 << 17]byte + var x2523 [1 << 17]byte + var x2524 [1 << 17]byte + var x2525 [1 << 17]byte + var x2526 [1 << 17]byte + var x2527 [1 << 17]byte + var x2528 [1 << 17]byte + var x2529 [1 << 17]byte + var x2530 [1 << 17]byte + var x2531 [1 << 17]byte + var x2532 [1 << 17]byte + var x2533 [1 << 17]byte + var x2534 [1 << 17]byte + var x2535 [1 << 17]byte + var x2536 [1 << 17]byte + var x2537 [1 << 17]byte + var x2538 [1 << 17]byte + var x2539 [1 << 17]byte + var x2540 [1 << 17]byte + var x2541 [1 << 17]byte + var x2542 [1 << 17]byte + var x2543 [1 << 17]byte + var x2544 [1 << 17]byte + var x2545 [1 << 17]byte + var x2546 [1 << 17]byte + var x2547 [1 << 17]byte + var x2548 [1 << 17]byte + var x2549 [1 << 17]byte + var x2550 [1 << 17]byte + var x2551 [1 << 17]byte + var x2552 [1 << 17]byte + var x2553 [1 << 17]byte + var x2554 [1 << 17]byte + var x2555 [1 << 17]byte + var x2556 [1 << 17]byte + var x2557 [1 << 17]byte + var x2558 [1 << 17]byte + var x2559 [1 << 17]byte + var x2560 [1 << 17]byte + var x2561 [1 << 17]byte + var x2562 [1 << 17]byte + var x2563 [1 << 17]byte + var x2564 [1 << 17]byte + var x2565 [1 << 17]byte + var x2566 [1 << 17]byte + var x2567 [1 << 17]byte + var x2568 [1 << 17]byte + var x2569 [1 << 17]byte + var x2570 [1 << 17]byte + var x2571 [1 << 17]byte + var x2572 [1 << 17]byte + var x2573 [1 << 17]byte + var x2574 [1 << 17]byte + var x2575 [1 << 17]byte + var x2576 [1 << 17]byte + var x2577 [1 << 17]byte + var x2578 [1 << 17]byte + var x2579 [1 << 17]byte + var x2580 [1 << 17]byte + var x2581 [1 << 17]byte + var x2582 [1 << 17]byte + var x2583 [1 << 17]byte + var x2584 [1 << 17]byte + var x2585 [1 << 17]byte + var x2586 [1 << 17]byte + var x2587 [1 << 17]byte + var x2588 [1 << 17]byte + var x2589 [1 << 17]byte + var x2590 [1 << 17]byte + var x2591 [1 << 17]byte + var x2592 [1 << 17]byte + var x2593 [1 << 17]byte + var x2594 [1 << 17]byte + var x2595 [1 << 17]byte + var x2596 [1 << 17]byte + var x2597 [1 << 17]byte + var x2598 [1 << 17]byte + var x2599 [1 << 17]byte + var x2600 [1 << 17]byte + var x2601 [1 << 17]byte + var x2602 [1 << 17]byte + var x2603 [1 << 17]byte + var x2604 [1 << 17]byte + var x2605 [1 << 17]byte + var x2606 [1 << 17]byte + var x2607 [1 << 17]byte + var x2608 [1 << 17]byte + var x2609 [1 << 17]byte + var x2610 [1 << 17]byte + var x2611 [1 << 17]byte + var x2612 [1 << 17]byte + var x2613 [1 << 17]byte + var x2614 [1 << 17]byte + var x2615 [1 << 17]byte + var x2616 [1 << 17]byte + var x2617 [1 << 17]byte + var x2618 [1 << 17]byte + var x2619 [1 << 17]byte + var x2620 [1 << 17]byte + var x2621 [1 << 17]byte + var x2622 [1 << 17]byte + var x2623 [1 << 17]byte + var x2624 [1 << 17]byte + var x2625 [1 << 17]byte + var x2626 [1 << 17]byte + var x2627 [1 << 17]byte + var x2628 [1 << 17]byte + var x2629 [1 << 17]byte + var x2630 [1 << 17]byte + var x2631 [1 << 17]byte + var x2632 [1 << 17]byte + var x2633 [1 << 17]byte + var x2634 [1 << 17]byte + var x2635 [1 << 17]byte + var x2636 [1 << 17]byte + var x2637 [1 << 17]byte + var x2638 [1 << 17]byte + var x2639 [1 << 17]byte + var x2640 [1 << 17]byte + var x2641 [1 << 17]byte + var x2642 [1 << 17]byte + var x2643 [1 << 17]byte + var x2644 [1 << 17]byte + var x2645 [1 << 17]byte + var x2646 [1 << 17]byte + var x2647 [1 << 17]byte + var x2648 [1 << 17]byte + var x2649 [1 << 17]byte + var x2650 [1 << 17]byte + var x2651 [1 << 17]byte + var x2652 [1 << 17]byte + var x2653 [1 << 17]byte + var x2654 [1 << 17]byte + var x2655 [1 << 17]byte + var x2656 [1 << 17]byte + var x2657 [1 << 17]byte + var x2658 [1 << 17]byte + var x2659 [1 << 17]byte + var x2660 [1 << 17]byte + var x2661 [1 << 17]byte + var x2662 [1 << 17]byte + var x2663 [1 << 17]byte + var x2664 [1 << 17]byte + var x2665 [1 << 17]byte + var x2666 [1 << 17]byte + var x2667 [1 << 17]byte + var x2668 [1 << 17]byte + var x2669 [1 << 17]byte + var x2670 [1 << 17]byte + var x2671 [1 << 17]byte + var x2672 [1 << 17]byte + var x2673 [1 << 17]byte + var x2674 [1 << 17]byte + var x2675 [1 << 17]byte + var x2676 [1 << 17]byte + var x2677 [1 << 17]byte + var x2678 [1 << 17]byte + var x2679 [1 << 17]byte + var x2680 [1 << 17]byte + var x2681 [1 << 17]byte + var x2682 [1 << 17]byte + var x2683 [1 << 17]byte + var x2684 [1 << 17]byte + var x2685 [1 << 17]byte + var x2686 [1 << 17]byte + var x2687 [1 << 17]byte + var x2688 [1 << 17]byte + var x2689 [1 << 17]byte + var x2690 [1 << 17]byte + var x2691 [1 << 17]byte + var x2692 [1 << 17]byte + var x2693 [1 << 17]byte + var x2694 [1 << 17]byte + var x2695 [1 << 17]byte + var x2696 [1 << 17]byte + var x2697 [1 << 17]byte + var x2698 [1 << 17]byte + var x2699 [1 << 17]byte + var x2700 [1 << 17]byte + var x2701 [1 << 17]byte + var x2702 [1 << 17]byte + var x2703 [1 << 17]byte + var x2704 [1 << 17]byte + var x2705 [1 << 17]byte + var x2706 [1 << 17]byte + var x2707 [1 << 17]byte + var x2708 [1 << 17]byte + var x2709 [1 << 17]byte + var x2710 [1 << 17]byte + var x2711 [1 << 17]byte + var x2712 [1 << 17]byte + var x2713 [1 << 17]byte + var x2714 [1 << 17]byte + var x2715 [1 << 17]byte + var x2716 [1 << 17]byte + var x2717 [1 << 17]byte + var x2718 [1 << 17]byte + var x2719 [1 << 17]byte + var x2720 [1 << 17]byte + var x2721 [1 << 17]byte + var x2722 [1 << 17]byte + var x2723 [1 << 17]byte + var x2724 [1 << 17]byte + var x2725 [1 << 17]byte + var x2726 [1 << 17]byte + var x2727 [1 << 17]byte + var x2728 [1 << 17]byte + var x2729 [1 << 17]byte + var x2730 [1 << 17]byte + var x2731 [1 << 17]byte + var x2732 [1 << 17]byte + var x2733 [1 << 17]byte + var x2734 [1 << 17]byte + var x2735 [1 << 17]byte + var x2736 [1 << 17]byte + var x2737 [1 << 17]byte + var x2738 [1 << 17]byte + var x2739 [1 << 17]byte + var x2740 [1 << 17]byte + var x2741 [1 << 17]byte + var x2742 [1 << 17]byte + var x2743 [1 << 17]byte + var x2744 [1 << 17]byte + var x2745 [1 << 17]byte + var x2746 [1 << 17]byte + var x2747 [1 << 17]byte + var x2748 [1 << 17]byte + var x2749 [1 << 17]byte + var x2750 [1 << 17]byte + var x2751 [1 << 17]byte + var x2752 [1 << 17]byte + var x2753 [1 << 17]byte + var x2754 [1 << 17]byte + var x2755 [1 << 17]byte + var x2756 [1 << 17]byte + var x2757 [1 << 17]byte + var x2758 [1 << 17]byte + var x2759 [1 << 17]byte + var x2760 [1 << 17]byte + var x2761 [1 << 17]byte + var x2762 [1 << 17]byte + var x2763 [1 << 17]byte + var x2764 [1 << 17]byte + var x2765 [1 << 17]byte + var x2766 [1 << 17]byte + var x2767 [1 << 17]byte + var x2768 [1 << 17]byte + var x2769 [1 << 17]byte + var x2770 [1 << 17]byte + var x2771 [1 << 17]byte + var x2772 [1 << 17]byte + var x2773 [1 << 17]byte + var x2774 [1 << 17]byte + var x2775 [1 << 17]byte + var x2776 [1 << 17]byte + var x2777 [1 << 17]byte + var x2778 [1 << 17]byte + var x2779 [1 << 17]byte + var x2780 [1 << 17]byte + var x2781 [1 << 17]byte + var x2782 [1 << 17]byte + var x2783 [1 << 17]byte + var x2784 [1 << 17]byte + var x2785 [1 << 17]byte + var x2786 [1 << 17]byte + var x2787 [1 << 17]byte + var x2788 [1 << 17]byte + var x2789 [1 << 17]byte + var x2790 [1 << 17]byte + var x2791 [1 << 17]byte + var x2792 [1 << 17]byte + var x2793 [1 << 17]byte + var x2794 [1 << 17]byte + var x2795 [1 << 17]byte + var x2796 [1 << 17]byte + var x2797 [1 << 17]byte + var x2798 [1 << 17]byte + var x2799 [1 << 17]byte + var x2800 [1 << 17]byte + var x2801 [1 << 17]byte + var x2802 [1 << 17]byte + var x2803 [1 << 17]byte + var x2804 [1 << 17]byte + var x2805 [1 << 17]byte + var x2806 [1 << 17]byte + var x2807 [1 << 17]byte + var x2808 [1 << 17]byte + var x2809 [1 << 17]byte + var x2810 [1 << 17]byte + var x2811 [1 << 17]byte + var x2812 [1 << 17]byte + var x2813 [1 << 17]byte + var x2814 [1 << 17]byte + var x2815 [1 << 17]byte + var x2816 [1 << 17]byte + var x2817 [1 << 17]byte + var x2818 [1 << 17]byte + var x2819 [1 << 17]byte + var x2820 [1 << 17]byte + var x2821 [1 << 17]byte + var x2822 [1 << 17]byte + var x2823 [1 << 17]byte + var x2824 [1 << 17]byte + var x2825 [1 << 17]byte + var x2826 [1 << 17]byte + var x2827 [1 << 17]byte + var x2828 [1 << 17]byte + var x2829 [1 << 17]byte + var x2830 [1 << 17]byte + var x2831 [1 << 17]byte + var x2832 [1 << 17]byte + var x2833 [1 << 17]byte + var x2834 [1 << 17]byte + var x2835 [1 << 17]byte + var x2836 [1 << 17]byte + var x2837 [1 << 17]byte + var x2838 [1 << 17]byte + var x2839 [1 << 17]byte + var x2840 [1 << 17]byte + var x2841 [1 << 17]byte + var x2842 [1 << 17]byte + var x2843 [1 << 17]byte + var x2844 [1 << 17]byte + var x2845 [1 << 17]byte + var x2846 [1 << 17]byte + var x2847 [1 << 17]byte + var x2848 [1 << 17]byte + var x2849 [1 << 17]byte + var x2850 [1 << 17]byte + var x2851 [1 << 17]byte + var x2852 [1 << 17]byte + var x2853 [1 << 17]byte + var x2854 [1 << 17]byte + var x2855 [1 << 17]byte + var x2856 [1 << 17]byte + var x2857 [1 << 17]byte + var x2858 [1 << 17]byte + var x2859 [1 << 17]byte + var x2860 [1 << 17]byte + var x2861 [1 << 17]byte + var x2862 [1 << 17]byte + var x2863 [1 << 17]byte + var x2864 [1 << 17]byte + var x2865 [1 << 17]byte + var x2866 [1 << 17]byte + var x2867 [1 << 17]byte + var x2868 [1 << 17]byte + var x2869 [1 << 17]byte + var x2870 [1 << 17]byte + var x2871 [1 << 17]byte + var x2872 [1 << 17]byte + var x2873 [1 << 17]byte + var x2874 [1 << 17]byte + var x2875 [1 << 17]byte + var x2876 [1 << 17]byte + var x2877 [1 << 17]byte + var x2878 [1 << 17]byte + var x2879 [1 << 17]byte + var x2880 [1 << 17]byte + var x2881 [1 << 17]byte + var x2882 [1 << 17]byte + var x2883 [1 << 17]byte + var x2884 [1 << 17]byte + var x2885 [1 << 17]byte + var x2886 [1 << 17]byte + var x2887 [1 << 17]byte + var x2888 [1 << 17]byte + var x2889 [1 << 17]byte + var x2890 [1 << 17]byte + var x2891 [1 << 17]byte + var x2892 [1 << 17]byte + var x2893 [1 << 17]byte + var x2894 [1 << 17]byte + var x2895 [1 << 17]byte + var x2896 [1 << 17]byte + var x2897 [1 << 17]byte + var x2898 [1 << 17]byte + var x2899 [1 << 17]byte + var x2900 [1 << 17]byte + var x2901 [1 << 17]byte + var x2902 [1 << 17]byte + var x2903 [1 << 17]byte + var x2904 [1 << 17]byte + var x2905 [1 << 17]byte + var x2906 [1 << 17]byte + var x2907 [1 << 17]byte + var x2908 [1 << 17]byte + var x2909 [1 << 17]byte + var x2910 [1 << 17]byte + var x2911 [1 << 17]byte + var x2912 [1 << 17]byte + var x2913 [1 << 17]byte + var x2914 [1 << 17]byte + var x2915 [1 << 17]byte + var x2916 [1 << 17]byte + var x2917 [1 << 17]byte + var x2918 [1 << 17]byte + var x2919 [1 << 17]byte + var x2920 [1 << 17]byte + var x2921 [1 << 17]byte + var x2922 [1 << 17]byte + var x2923 [1 << 17]byte + var x2924 [1 << 17]byte + var x2925 [1 << 17]byte + var x2926 [1 << 17]byte + var x2927 [1 << 17]byte + var x2928 [1 << 17]byte + var x2929 [1 << 17]byte + var x2930 [1 << 17]byte + var x2931 [1 << 17]byte + var x2932 [1 << 17]byte + var x2933 [1 << 17]byte + var x2934 [1 << 17]byte + var x2935 [1 << 17]byte + var x2936 [1 << 17]byte + var x2937 [1 << 17]byte + var x2938 [1 << 17]byte + var x2939 [1 << 17]byte + var x2940 [1 << 17]byte + var x2941 [1 << 17]byte + var x2942 [1 << 17]byte + var x2943 [1 << 17]byte + var x2944 [1 << 17]byte + var x2945 [1 << 17]byte + var x2946 [1 << 17]byte + var x2947 [1 << 17]byte + var x2948 [1 << 17]byte + var x2949 [1 << 17]byte + var x2950 [1 << 17]byte + var x2951 [1 << 17]byte + var x2952 [1 << 17]byte + var x2953 [1 << 17]byte + var x2954 [1 << 17]byte + var x2955 [1 << 17]byte + var x2956 [1 << 17]byte + var x2957 [1 << 17]byte + var x2958 [1 << 17]byte + var x2959 [1 << 17]byte + var x2960 [1 << 17]byte + var x2961 [1 << 17]byte + var x2962 [1 << 17]byte + var x2963 [1 << 17]byte + var x2964 [1 << 17]byte + var x2965 [1 << 17]byte + var x2966 [1 << 17]byte + var x2967 [1 << 17]byte + var x2968 [1 << 17]byte + var x2969 [1 << 17]byte + var x2970 [1 << 17]byte + var x2971 [1 << 17]byte + var x2972 [1 << 17]byte + var x2973 [1 << 17]byte + var x2974 [1 << 17]byte + var x2975 [1 << 17]byte + var x2976 [1 << 17]byte + var x2977 [1 << 17]byte + var x2978 [1 << 17]byte + var x2979 [1 << 17]byte + var x2980 [1 << 17]byte + var x2981 [1 << 17]byte + var x2982 [1 << 17]byte + var x2983 [1 << 17]byte + var x2984 [1 << 17]byte + var x2985 [1 << 17]byte + var x2986 [1 << 17]byte + var x2987 [1 << 17]byte + var x2988 [1 << 17]byte + var x2989 [1 << 17]byte + var x2990 [1 << 17]byte + var x2991 [1 << 17]byte + var x2992 [1 << 17]byte + var x2993 [1 << 17]byte + var x2994 [1 << 17]byte + var x2995 [1 << 17]byte + var x2996 [1 << 17]byte + var x2997 [1 << 17]byte + var x2998 [1 << 17]byte + var x2999 [1 << 17]byte + var x3000 [1 << 17]byte + var x3001 [1 << 17]byte + var x3002 [1 << 17]byte + var x3003 [1 << 17]byte + var x3004 [1 << 17]byte + var x3005 [1 << 17]byte + var x3006 [1 << 17]byte + var x3007 [1 << 17]byte + var x3008 [1 << 17]byte + var x3009 [1 << 17]byte + var x3010 [1 << 17]byte + var x3011 [1 << 17]byte + var x3012 [1 << 17]byte + var x3013 [1 << 17]byte + var x3014 [1 << 17]byte + var x3015 [1 << 17]byte + var x3016 [1 << 17]byte + var x3017 [1 << 17]byte + var x3018 [1 << 17]byte + var x3019 [1 << 17]byte + var x3020 [1 << 17]byte + var x3021 [1 << 17]byte + var x3022 [1 << 17]byte + var x3023 [1 << 17]byte + var x3024 [1 << 17]byte + var x3025 [1 << 17]byte + var x3026 [1 << 17]byte + var x3027 [1 << 17]byte + var x3028 [1 << 17]byte + var x3029 [1 << 17]byte + var x3030 [1 << 17]byte + var x3031 [1 << 17]byte + var x3032 [1 << 17]byte + var x3033 [1 << 17]byte + var x3034 [1 << 17]byte + var x3035 [1 << 17]byte + var x3036 [1 << 17]byte + var x3037 [1 << 17]byte + var x3038 [1 << 17]byte + var x3039 [1 << 17]byte + var x3040 [1 << 17]byte + var x3041 [1 << 17]byte + var x3042 [1 << 17]byte + var x3043 [1 << 17]byte + var x3044 [1 << 17]byte + var x3045 [1 << 17]byte + var x3046 [1 << 17]byte + var x3047 [1 << 17]byte + var x3048 [1 << 17]byte + var x3049 [1 << 17]byte + var x3050 [1 << 17]byte + var x3051 [1 << 17]byte + var x3052 [1 << 17]byte + var x3053 [1 << 17]byte + var x3054 [1 << 17]byte + var x3055 [1 << 17]byte + var x3056 [1 << 17]byte + var x3057 [1 << 17]byte + var x3058 [1 << 17]byte + var x3059 [1 << 17]byte + var x3060 [1 << 17]byte + var x3061 [1 << 17]byte + var x3062 [1 << 17]byte + var x3063 [1 << 17]byte + var x3064 [1 << 17]byte + var x3065 [1 << 17]byte + var x3066 [1 << 17]byte + var x3067 [1 << 17]byte + var x3068 [1 << 17]byte + var x3069 [1 << 17]byte + var x3070 [1 << 17]byte + var x3071 [1 << 17]byte + var x3072 [1 << 17]byte + var x3073 [1 << 17]byte + var x3074 [1 << 17]byte + var x3075 [1 << 17]byte + var x3076 [1 << 17]byte + var x3077 [1 << 17]byte + var x3078 [1 << 17]byte + var x3079 [1 << 17]byte + var x3080 [1 << 17]byte + var x3081 [1 << 17]byte + var x3082 [1 << 17]byte + var x3083 [1 << 17]byte + var x3084 [1 << 17]byte + var x3085 [1 << 17]byte + var x3086 [1 << 17]byte + var x3087 [1 << 17]byte + var x3088 [1 << 17]byte + var x3089 [1 << 17]byte + var x3090 [1 << 17]byte + var x3091 [1 << 17]byte + var x3092 [1 << 17]byte + var x3093 [1 << 17]byte + var x3094 [1 << 17]byte + var x3095 [1 << 17]byte + var x3096 [1 << 17]byte + var x3097 [1 << 17]byte + var x3098 [1 << 17]byte + var x3099 [1 << 17]byte + var x3100 [1 << 17]byte + var x3101 [1 << 17]byte + var x3102 [1 << 17]byte + var x3103 [1 << 17]byte + var x3104 [1 << 17]byte + var x3105 [1 << 17]byte + var x3106 [1 << 17]byte + var x3107 [1 << 17]byte + var x3108 [1 << 17]byte + var x3109 [1 << 17]byte + var x3110 [1 << 17]byte + var x3111 [1 << 17]byte + var x3112 [1 << 17]byte + var x3113 [1 << 17]byte + var x3114 [1 << 17]byte + var x3115 [1 << 17]byte + var x3116 [1 << 17]byte + var x3117 [1 << 17]byte + var x3118 [1 << 17]byte + var x3119 [1 << 17]byte + var x3120 [1 << 17]byte + var x3121 [1 << 17]byte + var x3122 [1 << 17]byte + var x3123 [1 << 17]byte + var x3124 [1 << 17]byte + var x3125 [1 << 17]byte + var x3126 [1 << 17]byte + var x3127 [1 << 17]byte + var x3128 [1 << 17]byte + var x3129 [1 << 17]byte + var x3130 [1 << 17]byte + var x3131 [1 << 17]byte + var x3132 [1 << 17]byte + var x3133 [1 << 17]byte + var x3134 [1 << 17]byte + var x3135 [1 << 17]byte + var x3136 [1 << 17]byte + var x3137 [1 << 17]byte + var x3138 [1 << 17]byte + var x3139 [1 << 17]byte + var x3140 [1 << 17]byte + var x3141 [1 << 17]byte + var x3142 [1 << 17]byte + var x3143 [1 << 17]byte + var x3144 [1 << 17]byte + var x3145 [1 << 17]byte + var x3146 [1 << 17]byte + var x3147 [1 << 17]byte + var x3148 [1 << 17]byte + var x3149 [1 << 17]byte + var x3150 [1 << 17]byte + var x3151 [1 << 17]byte + var x3152 [1 << 17]byte + var x3153 [1 << 17]byte + var x3154 [1 << 17]byte + var x3155 [1 << 17]byte + var x3156 [1 << 17]byte + var x3157 [1 << 17]byte + var x3158 [1 << 17]byte + var x3159 [1 << 17]byte + var x3160 [1 << 17]byte + var x3161 [1 << 17]byte + var x3162 [1 << 17]byte + var x3163 [1 << 17]byte + var x3164 [1 << 17]byte + var x3165 [1 << 17]byte + var x3166 [1 << 17]byte + var x3167 [1 << 17]byte + var x3168 [1 << 17]byte + var x3169 [1 << 17]byte + var x3170 [1 << 17]byte + var x3171 [1 << 17]byte + var x3172 [1 << 17]byte + var x3173 [1 << 17]byte + var x3174 [1 << 17]byte + var x3175 [1 << 17]byte + var x3176 [1 << 17]byte + var x3177 [1 << 17]byte + var x3178 [1 << 17]byte + var x3179 [1 << 17]byte + var x3180 [1 << 17]byte + var x3181 [1 << 17]byte + var x3182 [1 << 17]byte + var x3183 [1 << 17]byte + var x3184 [1 << 17]byte + var x3185 [1 << 17]byte + var x3186 [1 << 17]byte + var x3187 [1 << 17]byte + var x3188 [1 << 17]byte + var x3189 [1 << 17]byte + var x3190 [1 << 17]byte + var x3191 [1 << 17]byte + var x3192 [1 << 17]byte + var x3193 [1 << 17]byte + var x3194 [1 << 17]byte + var x3195 [1 << 17]byte + var x3196 [1 << 17]byte + var x3197 [1 << 17]byte + var x3198 [1 << 17]byte + var x3199 [1 << 17]byte + var x3200 [1 << 17]byte + var x3201 [1 << 17]byte + var x3202 [1 << 17]byte + var x3203 [1 << 17]byte + var x3204 [1 << 17]byte + var x3205 [1 << 17]byte + var x3206 [1 << 17]byte + var x3207 [1 << 17]byte + var x3208 [1 << 17]byte + var x3209 [1 << 17]byte + var x3210 [1 << 17]byte + var x3211 [1 << 17]byte + var x3212 [1 << 17]byte + var x3213 [1 << 17]byte + var x3214 [1 << 17]byte + var x3215 [1 << 17]byte + var x3216 [1 << 17]byte + var x3217 [1 << 17]byte + var x3218 [1 << 17]byte + var x3219 [1 << 17]byte + var x3220 [1 << 17]byte + var x3221 [1 << 17]byte + var x3222 [1 << 17]byte + var x3223 [1 << 17]byte + var x3224 [1 << 17]byte + var x3225 [1 << 17]byte + var x3226 [1 << 17]byte + var x3227 [1 << 17]byte + var x3228 [1 << 17]byte + var x3229 [1 << 17]byte + var x3230 [1 << 17]byte + var x3231 [1 << 17]byte + var x3232 [1 << 17]byte + var x3233 [1 << 17]byte + var x3234 [1 << 17]byte + var x3235 [1 << 17]byte + var x3236 [1 << 17]byte + var x3237 [1 << 17]byte + var x3238 [1 << 17]byte + var x3239 [1 << 17]byte + var x3240 [1 << 17]byte + var x3241 [1 << 17]byte + var x3242 [1 << 17]byte + var x3243 [1 << 17]byte + var x3244 [1 << 17]byte + var x3245 [1 << 17]byte + var x3246 [1 << 17]byte + var x3247 [1 << 17]byte + var x3248 [1 << 17]byte + var x3249 [1 << 17]byte + var x3250 [1 << 17]byte + var x3251 [1 << 17]byte + var x3252 [1 << 17]byte + var x3253 [1 << 17]byte + var x3254 [1 << 17]byte + var x3255 [1 << 17]byte + var x3256 [1 << 17]byte + var x3257 [1 << 17]byte + var x3258 [1 << 17]byte + var x3259 [1 << 17]byte + var x3260 [1 << 17]byte + var x3261 [1 << 17]byte + var x3262 [1 << 17]byte + var x3263 [1 << 17]byte + var x3264 [1 << 17]byte + var x3265 [1 << 17]byte + var x3266 [1 << 17]byte + var x3267 [1 << 17]byte + var x3268 [1 << 17]byte + var x3269 [1 << 17]byte + var x3270 [1 << 17]byte + var x3271 [1 << 17]byte + var x3272 [1 << 17]byte + var x3273 [1 << 17]byte + var x3274 [1 << 17]byte + var x3275 [1 << 17]byte + var x3276 [1 << 17]byte + var x3277 [1 << 17]byte + var x3278 [1 << 17]byte + var x3279 [1 << 17]byte + var x3280 [1 << 17]byte + var x3281 [1 << 17]byte + var x3282 [1 << 17]byte + var x3283 [1 << 17]byte + var x3284 [1 << 17]byte + var x3285 [1 << 17]byte + var x3286 [1 << 17]byte + var x3287 [1 << 17]byte + var x3288 [1 << 17]byte + var x3289 [1 << 17]byte + var x3290 [1 << 17]byte + var x3291 [1 << 17]byte + var x3292 [1 << 17]byte + var x3293 [1 << 17]byte + var x3294 [1 << 17]byte + var x3295 [1 << 17]byte + var x3296 [1 << 17]byte + var x3297 [1 << 17]byte + var x3298 [1 << 17]byte + var x3299 [1 << 17]byte + var x3300 [1 << 17]byte + var x3301 [1 << 17]byte + var x3302 [1 << 17]byte + var x3303 [1 << 17]byte + var x3304 [1 << 17]byte + var x3305 [1 << 17]byte + var x3306 [1 << 17]byte + var x3307 [1 << 17]byte + var x3308 [1 << 17]byte + var x3309 [1 << 17]byte + var x3310 [1 << 17]byte + var x3311 [1 << 17]byte + var x3312 [1 << 17]byte + var x3313 [1 << 17]byte + var x3314 [1 << 17]byte + var x3315 [1 << 17]byte + var x3316 [1 << 17]byte + var x3317 [1 << 17]byte + var x3318 [1 << 17]byte + var x3319 [1 << 17]byte + var x3320 [1 << 17]byte + var x3321 [1 << 17]byte + var x3322 [1 << 17]byte + var x3323 [1 << 17]byte + var x3324 [1 << 17]byte + var x3325 [1 << 17]byte + var x3326 [1 << 17]byte + var x3327 [1 << 17]byte + var x3328 [1 << 17]byte + var x3329 [1 << 17]byte + var x3330 [1 << 17]byte + var x3331 [1 << 17]byte + var x3332 [1 << 17]byte + var x3333 [1 << 17]byte + var x3334 [1 << 17]byte + var x3335 [1 << 17]byte + var x3336 [1 << 17]byte + var x3337 [1 << 17]byte + var x3338 [1 << 17]byte + var x3339 [1 << 17]byte + var x3340 [1 << 17]byte + var x3341 [1 << 17]byte + var x3342 [1 << 17]byte + var x3343 [1 << 17]byte + var x3344 [1 << 17]byte + var x3345 [1 << 17]byte + var x3346 [1 << 17]byte + var x3347 [1 << 17]byte + var x3348 [1 << 17]byte + var x3349 [1 << 17]byte + var x3350 [1 << 17]byte + var x3351 [1 << 17]byte + var x3352 [1 << 17]byte + var x3353 [1 << 17]byte + var x3354 [1 << 17]byte + var x3355 [1 << 17]byte + var x3356 [1 << 17]byte + var x3357 [1 << 17]byte + var x3358 [1 << 17]byte + var x3359 [1 << 17]byte + var x3360 [1 << 17]byte + var x3361 [1 << 17]byte + var x3362 [1 << 17]byte + var x3363 [1 << 17]byte + var x3364 [1 << 17]byte + var x3365 [1 << 17]byte + var x3366 [1 << 17]byte + var x3367 [1 << 17]byte + var x3368 [1 << 17]byte + var x3369 [1 << 17]byte + var x3370 [1 << 17]byte + var x3371 [1 << 17]byte + var x3372 [1 << 17]byte + var x3373 [1 << 17]byte + var x3374 [1 << 17]byte + var x3375 [1 << 17]byte + var x3376 [1 << 17]byte + var x3377 [1 << 17]byte + var x3378 [1 << 17]byte + var x3379 [1 << 17]byte + var x3380 [1 << 17]byte + var x3381 [1 << 17]byte + var x3382 [1 << 17]byte + var x3383 [1 << 17]byte + var x3384 [1 << 17]byte + var x3385 [1 << 17]byte + var x3386 [1 << 17]byte + var x3387 [1 << 17]byte + var x3388 [1 << 17]byte + var x3389 [1 << 17]byte + var x3390 [1 << 17]byte + var x3391 [1 << 17]byte + var x3392 [1 << 17]byte + var x3393 [1 << 17]byte + var x3394 [1 << 17]byte + var x3395 [1 << 17]byte + var x3396 [1 << 17]byte + var x3397 [1 << 17]byte + var x3398 [1 << 17]byte + var x3399 [1 << 17]byte + var x3400 [1 << 17]byte + var x3401 [1 << 17]byte + var x3402 [1 << 17]byte + var x3403 [1 << 17]byte + var x3404 [1 << 17]byte + var x3405 [1 << 17]byte + var x3406 [1 << 17]byte + var x3407 [1 << 17]byte + var x3408 [1 << 17]byte + var x3409 [1 << 17]byte + var x3410 [1 << 17]byte + var x3411 [1 << 17]byte + var x3412 [1 << 17]byte + var x3413 [1 << 17]byte + var x3414 [1 << 17]byte + var x3415 [1 << 17]byte + var x3416 [1 << 17]byte + var x3417 [1 << 17]byte + var x3418 [1 << 17]byte + var x3419 [1 << 17]byte + var x3420 [1 << 17]byte + var x3421 [1 << 17]byte + var x3422 [1 << 17]byte + var x3423 [1 << 17]byte + var x3424 [1 << 17]byte + var x3425 [1 << 17]byte + var x3426 [1 << 17]byte + var x3427 [1 << 17]byte + var x3428 [1 << 17]byte + var x3429 [1 << 17]byte + var x3430 [1 << 17]byte + var x3431 [1 << 17]byte + var x3432 [1 << 17]byte + var x3433 [1 << 17]byte + var x3434 [1 << 17]byte + var x3435 [1 << 17]byte + var x3436 [1 << 17]byte + var x3437 [1 << 17]byte + var x3438 [1 << 17]byte + var x3439 [1 << 17]byte + var x3440 [1 << 17]byte + var x3441 [1 << 17]byte + var x3442 [1 << 17]byte + var x3443 [1 << 17]byte + var x3444 [1 << 17]byte + var x3445 [1 << 17]byte + var x3446 [1 << 17]byte + var x3447 [1 << 17]byte + var x3448 [1 << 17]byte + var x3449 [1 << 17]byte + var x3450 [1 << 17]byte + var x3451 [1 << 17]byte + var x3452 [1 << 17]byte + var x3453 [1 << 17]byte + var x3454 [1 << 17]byte + var x3455 [1 << 17]byte + var x3456 [1 << 17]byte + var x3457 [1 << 17]byte + var x3458 [1 << 17]byte + var x3459 [1 << 17]byte + var x3460 [1 << 17]byte + var x3461 [1 << 17]byte + var x3462 [1 << 17]byte + var x3463 [1 << 17]byte + var x3464 [1 << 17]byte + var x3465 [1 << 17]byte + var x3466 [1 << 17]byte + var x3467 [1 << 17]byte + var x3468 [1 << 17]byte + var x3469 [1 << 17]byte + var x3470 [1 << 17]byte + var x3471 [1 << 17]byte + var x3472 [1 << 17]byte + var x3473 [1 << 17]byte + var x3474 [1 << 17]byte + var x3475 [1 << 17]byte + var x3476 [1 << 17]byte + var x3477 [1 << 17]byte + var x3478 [1 << 17]byte + var x3479 [1 << 17]byte + var x3480 [1 << 17]byte + var x3481 [1 << 17]byte + var x3482 [1 << 17]byte + var x3483 [1 << 17]byte + var x3484 [1 << 17]byte + var x3485 [1 << 17]byte + var x3486 [1 << 17]byte + var x3487 [1 << 17]byte + var x3488 [1 << 17]byte + var x3489 [1 << 17]byte + var x3490 [1 << 17]byte + var x3491 [1 << 17]byte + var x3492 [1 << 17]byte + var x3493 [1 << 17]byte + var x3494 [1 << 17]byte + var x3495 [1 << 17]byte + var x3496 [1 << 17]byte + var x3497 [1 << 17]byte + var x3498 [1 << 17]byte + var x3499 [1 << 17]byte + var x3500 [1 << 17]byte + var x3501 [1 << 17]byte + var x3502 [1 << 17]byte + var x3503 [1 << 17]byte + var x3504 [1 << 17]byte + var x3505 [1 << 17]byte + var x3506 [1 << 17]byte + var x3507 [1 << 17]byte + var x3508 [1 << 17]byte + var x3509 [1 << 17]byte + var x3510 [1 << 17]byte + var x3511 [1 << 17]byte + var x3512 [1 << 17]byte + var x3513 [1 << 17]byte + var x3514 [1 << 17]byte + var x3515 [1 << 17]byte + var x3516 [1 << 17]byte + var x3517 [1 << 17]byte + var x3518 [1 << 17]byte + var x3519 [1 << 17]byte + var x3520 [1 << 17]byte + var x3521 [1 << 17]byte + var x3522 [1 << 17]byte + var x3523 [1 << 17]byte + var x3524 [1 << 17]byte + var x3525 [1 << 17]byte + var x3526 [1 << 17]byte + var x3527 [1 << 17]byte + var x3528 [1 << 17]byte + var x3529 [1 << 17]byte + var x3530 [1 << 17]byte + var x3531 [1 << 17]byte + var x3532 [1 << 17]byte + var x3533 [1 << 17]byte + var x3534 [1 << 17]byte + var x3535 [1 << 17]byte + var x3536 [1 << 17]byte + var x3537 [1 << 17]byte + var x3538 [1 << 17]byte + var x3539 [1 << 17]byte + var x3540 [1 << 17]byte + var x3541 [1 << 17]byte + var x3542 [1 << 17]byte + var x3543 [1 << 17]byte + var x3544 [1 << 17]byte + var x3545 [1 << 17]byte + var x3546 [1 << 17]byte + var x3547 [1 << 17]byte + var x3548 [1 << 17]byte + var x3549 [1 << 17]byte + var x3550 [1 << 17]byte + var x3551 [1 << 17]byte + var x3552 [1 << 17]byte + var x3553 [1 << 17]byte + var x3554 [1 << 17]byte + var x3555 [1 << 17]byte + var x3556 [1 << 17]byte + var x3557 [1 << 17]byte + var x3558 [1 << 17]byte + var x3559 [1 << 17]byte + var x3560 [1 << 17]byte + var x3561 [1 << 17]byte + var x3562 [1 << 17]byte + var x3563 [1 << 17]byte + var x3564 [1 << 17]byte + var x3565 [1 << 17]byte + var x3566 [1 << 17]byte + var x3567 [1 << 17]byte + var x3568 [1 << 17]byte + var x3569 [1 << 17]byte + var x3570 [1 << 17]byte + var x3571 [1 << 17]byte + var x3572 [1 << 17]byte + var x3573 [1 << 17]byte + var x3574 [1 << 17]byte + var x3575 [1 << 17]byte + var x3576 [1 << 17]byte + var x3577 [1 << 17]byte + var x3578 [1 << 17]byte + var x3579 [1 << 17]byte + var x3580 [1 << 17]byte + var x3581 [1 << 17]byte + var x3582 [1 << 17]byte + var x3583 [1 << 17]byte + var x3584 [1 << 17]byte + var x3585 [1 << 17]byte + var x3586 [1 << 17]byte + var x3587 [1 << 17]byte + var x3588 [1 << 17]byte + var x3589 [1 << 17]byte + var x3590 [1 << 17]byte + var x3591 [1 << 17]byte + var x3592 [1 << 17]byte + var x3593 [1 << 17]byte + var x3594 [1 << 17]byte + var x3595 [1 << 17]byte + var x3596 [1 << 17]byte + var x3597 [1 << 17]byte + var x3598 [1 << 17]byte + var x3599 [1 << 17]byte + var x3600 [1 << 17]byte + var x3601 [1 << 17]byte + var x3602 [1 << 17]byte + var x3603 [1 << 17]byte + var x3604 [1 << 17]byte + var x3605 [1 << 17]byte + var x3606 [1 << 17]byte + var x3607 [1 << 17]byte + var x3608 [1 << 17]byte + var x3609 [1 << 17]byte + var x3610 [1 << 17]byte + var x3611 [1 << 17]byte + var x3612 [1 << 17]byte + var x3613 [1 << 17]byte + var x3614 [1 << 17]byte + var x3615 [1 << 17]byte + var x3616 [1 << 17]byte + var x3617 [1 << 17]byte + var x3618 [1 << 17]byte + var x3619 [1 << 17]byte + var x3620 [1 << 17]byte + var x3621 [1 << 17]byte + var x3622 [1 << 17]byte + var x3623 [1 << 17]byte + var x3624 [1 << 17]byte + var x3625 [1 << 17]byte + var x3626 [1 << 17]byte + var x3627 [1 << 17]byte + var x3628 [1 << 17]byte + var x3629 [1 << 17]byte + var x3630 [1 << 17]byte + var x3631 [1 << 17]byte + var x3632 [1 << 17]byte + var x3633 [1 << 17]byte + var x3634 [1 << 17]byte + var x3635 [1 << 17]byte + var x3636 [1 << 17]byte + var x3637 [1 << 17]byte + var x3638 [1 << 17]byte + var x3639 [1 << 17]byte + var x3640 [1 << 17]byte + var x3641 [1 << 17]byte + var x3642 [1 << 17]byte + var x3643 [1 << 17]byte + var x3644 [1 << 17]byte + var x3645 [1 << 17]byte + var x3646 [1 << 17]byte + var x3647 [1 << 17]byte + var x3648 [1 << 17]byte + var x3649 [1 << 17]byte + var x3650 [1 << 17]byte + var x3651 [1 << 17]byte + var x3652 [1 << 17]byte + var x3653 [1 << 17]byte + var x3654 [1 << 17]byte + var x3655 [1 << 17]byte + var x3656 [1 << 17]byte + var x3657 [1 << 17]byte + var x3658 [1 << 17]byte + var x3659 [1 << 17]byte + var x3660 [1 << 17]byte + var x3661 [1 << 17]byte + var x3662 [1 << 17]byte + var x3663 [1 << 17]byte + var x3664 [1 << 17]byte + var x3665 [1 << 17]byte + var x3666 [1 << 17]byte + var x3667 [1 << 17]byte + var x3668 [1 << 17]byte + var x3669 [1 << 17]byte + var x3670 [1 << 17]byte + var x3671 [1 << 17]byte + var x3672 [1 << 17]byte + var x3673 [1 << 17]byte + var x3674 [1 << 17]byte + var x3675 [1 << 17]byte + var x3676 [1 << 17]byte + var x3677 [1 << 17]byte + var x3678 [1 << 17]byte + var x3679 [1 << 17]byte + var x3680 [1 << 17]byte + var x3681 [1 << 17]byte + var x3682 [1 << 17]byte + var x3683 [1 << 17]byte + var x3684 [1 << 17]byte + var x3685 [1 << 17]byte + var x3686 [1 << 17]byte + var x3687 [1 << 17]byte + var x3688 [1 << 17]byte + var x3689 [1 << 17]byte + var x3690 [1 << 17]byte + var x3691 [1 << 17]byte + var x3692 [1 << 17]byte + var x3693 [1 << 17]byte + var x3694 [1 << 17]byte + var x3695 [1 << 17]byte + var x3696 [1 << 17]byte + var x3697 [1 << 17]byte + var x3698 [1 << 17]byte + var x3699 [1 << 17]byte + var x3700 [1 << 17]byte + var x3701 [1 << 17]byte + var x3702 [1 << 17]byte + var x3703 [1 << 17]byte + var x3704 [1 << 17]byte + var x3705 [1 << 17]byte + var x3706 [1 << 17]byte + var x3707 [1 << 17]byte + var x3708 [1 << 17]byte + var x3709 [1 << 17]byte + var x3710 [1 << 17]byte + var x3711 [1 << 17]byte + var x3712 [1 << 17]byte + var x3713 [1 << 17]byte + var x3714 [1 << 17]byte + var x3715 [1 << 17]byte + var x3716 [1 << 17]byte + var x3717 [1 << 17]byte + var x3718 [1 << 17]byte + var x3719 [1 << 17]byte + var x3720 [1 << 17]byte + var x3721 [1 << 17]byte + var x3722 [1 << 17]byte + var x3723 [1 << 17]byte + var x3724 [1 << 17]byte + var x3725 [1 << 17]byte + var x3726 [1 << 17]byte + var x3727 [1 << 17]byte + var x3728 [1 << 17]byte + var x3729 [1 << 17]byte + var x3730 [1 << 17]byte + var x3731 [1 << 17]byte + var x3732 [1 << 17]byte + var x3733 [1 << 17]byte + var x3734 [1 << 17]byte + var x3735 [1 << 17]byte + var x3736 [1 << 17]byte + var x3737 [1 << 17]byte + var x3738 [1 << 17]byte + var x3739 [1 << 17]byte + var x3740 [1 << 17]byte + var x3741 [1 << 17]byte + var x3742 [1 << 17]byte + var x3743 [1 << 17]byte + var x3744 [1 << 17]byte + var x3745 [1 << 17]byte + var x3746 [1 << 17]byte + var x3747 [1 << 17]byte + var x3748 [1 << 17]byte + var x3749 [1 << 17]byte + var x3750 [1 << 17]byte + var x3751 [1 << 17]byte + var x3752 [1 << 17]byte + var x3753 [1 << 17]byte + var x3754 [1 << 17]byte + var x3755 [1 << 17]byte + var x3756 [1 << 17]byte + var x3757 [1 << 17]byte + var x3758 [1 << 17]byte + var x3759 [1 << 17]byte + var x3760 [1 << 17]byte + var x3761 [1 << 17]byte + var x3762 [1 << 17]byte + var x3763 [1 << 17]byte + var x3764 [1 << 17]byte + var x3765 [1 << 17]byte + var x3766 [1 << 17]byte + var x3767 [1 << 17]byte + var x3768 [1 << 17]byte + var x3769 [1 << 17]byte + var x3770 [1 << 17]byte + var x3771 [1 << 17]byte + var x3772 [1 << 17]byte + var x3773 [1 << 17]byte + var x3774 [1 << 17]byte + var x3775 [1 << 17]byte + var x3776 [1 << 17]byte + var x3777 [1 << 17]byte + var x3778 [1 << 17]byte + var x3779 [1 << 17]byte + var x3780 [1 << 17]byte + var x3781 [1 << 17]byte + var x3782 [1 << 17]byte + var x3783 [1 << 17]byte + var x3784 [1 << 17]byte + var x3785 [1 << 17]byte + var x3786 [1 << 17]byte + var x3787 [1 << 17]byte + var x3788 [1 << 17]byte + var x3789 [1 << 17]byte + var x3790 [1 << 17]byte + var x3791 [1 << 17]byte + var x3792 [1 << 17]byte + var x3793 [1 << 17]byte + var x3794 [1 << 17]byte + var x3795 [1 << 17]byte + var x3796 [1 << 17]byte + var x3797 [1 << 17]byte + var x3798 [1 << 17]byte + var x3799 [1 << 17]byte + var x3800 [1 << 17]byte + var x3801 [1 << 17]byte + var x3802 [1 << 17]byte + var x3803 [1 << 17]byte + var x3804 [1 << 17]byte + var x3805 [1 << 17]byte + var x3806 [1 << 17]byte + var x3807 [1 << 17]byte + var x3808 [1 << 17]byte + var x3809 [1 << 17]byte + var x3810 [1 << 17]byte + var x3811 [1 << 17]byte + var x3812 [1 << 17]byte + var x3813 [1 << 17]byte + var x3814 [1 << 17]byte + var x3815 [1 << 17]byte + var x3816 [1 << 17]byte + var x3817 [1 << 17]byte + var x3818 [1 << 17]byte + var x3819 [1 << 17]byte + var x3820 [1 << 17]byte + var x3821 [1 << 17]byte + var x3822 [1 << 17]byte + var x3823 [1 << 17]byte + var x3824 [1 << 17]byte + var x3825 [1 << 17]byte + var x3826 [1 << 17]byte + var x3827 [1 << 17]byte + var x3828 [1 << 17]byte + var x3829 [1 << 17]byte + var x3830 [1 << 17]byte + var x3831 [1 << 17]byte + var x3832 [1 << 17]byte + var x3833 [1 << 17]byte + var x3834 [1 << 17]byte + var x3835 [1 << 17]byte + var x3836 [1 << 17]byte + var x3837 [1 << 17]byte + var x3838 [1 << 17]byte + var x3839 [1 << 17]byte + var x3840 [1 << 17]byte + var x3841 [1 << 17]byte + var x3842 [1 << 17]byte + var x3843 [1 << 17]byte + var x3844 [1 << 17]byte + var x3845 [1 << 17]byte + var x3846 [1 << 17]byte + var x3847 [1 << 17]byte + var x3848 [1 << 17]byte + var x3849 [1 << 17]byte + var x3850 [1 << 17]byte + var x3851 [1 << 17]byte + var x3852 [1 << 17]byte + var x3853 [1 << 17]byte + var x3854 [1 << 17]byte + var x3855 [1 << 17]byte + var x3856 [1 << 17]byte + var x3857 [1 << 17]byte + var x3858 [1 << 17]byte + var x3859 [1 << 17]byte + var x3860 [1 << 17]byte + var x3861 [1 << 17]byte + var x3862 [1 << 17]byte + var x3863 [1 << 17]byte + var x3864 [1 << 17]byte + var x3865 [1 << 17]byte + var x3866 [1 << 17]byte + var x3867 [1 << 17]byte + var x3868 [1 << 17]byte + var x3869 [1 << 17]byte + var x3870 [1 << 17]byte + var x3871 [1 << 17]byte + var x3872 [1 << 17]byte + var x3873 [1 << 17]byte + var x3874 [1 << 17]byte + var x3875 [1 << 17]byte + var x3876 [1 << 17]byte + var x3877 [1 << 17]byte + var x3878 [1 << 17]byte + var x3879 [1 << 17]byte + var x3880 [1 << 17]byte + var x3881 [1 << 17]byte + var x3882 [1 << 17]byte + var x3883 [1 << 17]byte + var x3884 [1 << 17]byte + var x3885 [1 << 17]byte + var x3886 [1 << 17]byte + var x3887 [1 << 17]byte + var x3888 [1 << 17]byte + var x3889 [1 << 17]byte + var x3890 [1 << 17]byte + var x3891 [1 << 17]byte + var x3892 [1 << 17]byte + var x3893 [1 << 17]byte + var x3894 [1 << 17]byte + var x3895 [1 << 17]byte + var x3896 [1 << 17]byte + var x3897 [1 << 17]byte + var x3898 [1 << 17]byte + var x3899 [1 << 17]byte + var x3900 [1 << 17]byte + var x3901 [1 << 17]byte + var x3902 [1 << 17]byte + var x3903 [1 << 17]byte + var x3904 [1 << 17]byte + var x3905 [1 << 17]byte + var x3906 [1 << 17]byte + var x3907 [1 << 17]byte + var x3908 [1 << 17]byte + var x3909 [1 << 17]byte + var x3910 [1 << 17]byte + var x3911 [1 << 17]byte + var x3912 [1 << 17]byte + var x3913 [1 << 17]byte + var x3914 [1 << 17]byte + var x3915 [1 << 17]byte + var x3916 [1 << 17]byte + var x3917 [1 << 17]byte + var x3918 [1 << 17]byte + var x3919 [1 << 17]byte + var x3920 [1 << 17]byte + var x3921 [1 << 17]byte + var x3922 [1 << 17]byte + var x3923 [1 << 17]byte + var x3924 [1 << 17]byte + var x3925 [1 << 17]byte + var x3926 [1 << 17]byte + var x3927 [1 << 17]byte + var x3928 [1 << 17]byte + var x3929 [1 << 17]byte + var x3930 [1 << 17]byte + var x3931 [1 << 17]byte + var x3932 [1 << 17]byte + var x3933 [1 << 17]byte + var x3934 [1 << 17]byte + var x3935 [1 << 17]byte + var x3936 [1 << 17]byte + var x3937 [1 << 17]byte + var x3938 [1 << 17]byte + var x3939 [1 << 17]byte + var x3940 [1 << 17]byte + var x3941 [1 << 17]byte + var x3942 [1 << 17]byte + var x3943 [1 << 17]byte + var x3944 [1 << 17]byte + var x3945 [1 << 17]byte + var x3946 [1 << 17]byte + var x3947 [1 << 17]byte + var x3948 [1 << 17]byte + var x3949 [1 << 17]byte + var x3950 [1 << 17]byte + var x3951 [1 << 17]byte + var x3952 [1 << 17]byte + var x3953 [1 << 17]byte + var x3954 [1 << 17]byte + var x3955 [1 << 17]byte + var x3956 [1 << 17]byte + var x3957 [1 << 17]byte + var x3958 [1 << 17]byte + var x3959 [1 << 17]byte + var x3960 [1 << 17]byte + var x3961 [1 << 17]byte + var x3962 [1 << 17]byte + var x3963 [1 << 17]byte + var x3964 [1 << 17]byte + var x3965 [1 << 17]byte + var x3966 [1 << 17]byte + var x3967 [1 << 17]byte + var x3968 [1 << 17]byte + var x3969 [1 << 17]byte + var x3970 [1 << 17]byte + var x3971 [1 << 17]byte + var x3972 [1 << 17]byte + var x3973 [1 << 17]byte + var x3974 [1 << 17]byte + var x3975 [1 << 17]byte + var x3976 [1 << 17]byte + var x3977 [1 << 17]byte + var x3978 [1 << 17]byte + var x3979 [1 << 17]byte + var x3980 [1 << 17]byte + var x3981 [1 << 17]byte + var x3982 [1 << 17]byte + var x3983 [1 << 17]byte + var x3984 [1 << 17]byte + var x3985 [1 << 17]byte + var x3986 [1 << 17]byte + var x3987 [1 << 17]byte + var x3988 [1 << 17]byte + var x3989 [1 << 17]byte + var x3990 [1 << 17]byte + var x3991 [1 << 17]byte + var x3992 [1 << 17]byte + var x3993 [1 << 17]byte + var x3994 [1 << 17]byte + var x3995 [1 << 17]byte + var x3996 [1 << 17]byte + var x3997 [1 << 17]byte + var x3998 [1 << 17]byte + var x3999 [1 << 17]byte + var x4000 [1 << 17]byte + var x4001 [1 << 17]byte + var x4002 [1 << 17]byte + var x4003 [1 << 17]byte + var x4004 [1 << 17]byte + var x4005 [1 << 17]byte + var x4006 [1 << 17]byte + var x4007 [1 << 17]byte + var x4008 [1 << 17]byte + var x4009 [1 << 17]byte + var x4010 [1 << 17]byte + var x4011 [1 << 17]byte + var x4012 [1 << 17]byte + var x4013 [1 << 17]byte + var x4014 [1 << 17]byte + var x4015 [1 << 17]byte + var x4016 [1 << 17]byte + var x4017 [1 << 17]byte + var x4018 [1 << 17]byte + var x4019 [1 << 17]byte + var x4020 [1 << 17]byte + var x4021 [1 << 17]byte + var x4022 [1 << 17]byte + var x4023 [1 << 17]byte + var x4024 [1 << 17]byte + var x4025 [1 << 17]byte + var x4026 [1 << 17]byte + var x4027 [1 << 17]byte + var x4028 [1 << 17]byte + var x4029 [1 << 17]byte + var x4030 [1 << 17]byte + var x4031 [1 << 17]byte + var x4032 [1 << 17]byte + var x4033 [1 << 17]byte + var x4034 [1 << 17]byte + var x4035 [1 << 17]byte + var x4036 [1 << 17]byte + var x4037 [1 << 17]byte + var x4038 [1 << 17]byte + var x4039 [1 << 17]byte + var x4040 [1 << 17]byte + var x4041 [1 << 17]byte + var x4042 [1 << 17]byte + var x4043 [1 << 17]byte + var x4044 [1 << 17]byte + var x4045 [1 << 17]byte + var x4046 [1 << 17]byte + var x4047 [1 << 17]byte + var x4048 [1 << 17]byte + var x4049 [1 << 17]byte + var x4050 [1 << 17]byte + var x4051 [1 << 17]byte + var x4052 [1 << 17]byte + var x4053 [1 << 17]byte + var x4054 [1 << 17]byte + var x4055 [1 << 17]byte + var x4056 [1 << 17]byte + var x4057 [1 << 17]byte + var x4058 [1 << 17]byte + var x4059 [1 << 17]byte + var x4060 [1 << 17]byte + var x4061 [1 << 17]byte + var x4062 [1 << 17]byte + var x4063 [1 << 17]byte + var x4064 [1 << 17]byte + var x4065 [1 << 17]byte + var x4066 [1 << 17]byte + var x4067 [1 << 17]byte + var x4068 [1 << 17]byte + var x4069 [1 << 17]byte + var x4070 [1 << 17]byte + var x4071 [1 << 17]byte + var x4072 [1 << 17]byte + var x4073 [1 << 17]byte + var x4074 [1 << 17]byte + var x4075 [1 << 17]byte + var x4076 [1 << 17]byte + var x4077 [1 << 17]byte + var x4078 [1 << 17]byte + var x4079 [1 << 17]byte + var x4080 [1 << 17]byte + var x4081 [1 << 17]byte + var x4082 [1 << 17]byte + var x4083 [1 << 17]byte + var x4084 [1 << 17]byte + var x4085 [1 << 17]byte + var x4086 [1 << 17]byte + var x4087 [1 << 17]byte + var x4088 [1 << 17]byte + var x4089 [1 << 17]byte + var x4090 [1 << 17]byte + var x4091 [1 << 17]byte + var x4092 [1 << 17]byte + var x4093 [1 << 17]byte + var x4094 [1 << 17]byte + var x4095 [1 << 17]byte + var x4096 [1 << 17]byte + var x4097 [1 << 17]byte + var x4098 [1 << 17]byte + var x4099 [1 << 17]byte + var x4100 [1 << 17]byte + var x4101 [1 << 17]byte + var x4102 [1 << 17]byte + var x4103 [1 << 17]byte + var x4104 [1 << 17]byte + var x4105 [1 << 17]byte + var x4106 [1 << 17]byte + var x4107 [1 << 17]byte + var x4108 [1 << 17]byte + var x4109 [1 << 17]byte + var x4110 [1 << 17]byte + var x4111 [1 << 17]byte + var x4112 [1 << 17]byte + var x4113 [1 << 17]byte + var x4114 [1 << 17]byte + var x4115 [1 << 17]byte + var x4116 [1 << 17]byte + var x4117 [1 << 17]byte + var x4118 [1 << 17]byte + var x4119 [1 << 17]byte + var x4120 [1 << 17]byte + var x4121 [1 << 17]byte + var x4122 [1 << 17]byte + var x4123 [1 << 17]byte + var x4124 [1 << 17]byte + var x4125 [1 << 17]byte + var x4126 [1 << 17]byte + var x4127 [1 << 17]byte + var x4128 [1 << 17]byte + var x4129 [1 << 17]byte + var x4130 [1 << 17]byte + var x4131 [1 << 17]byte + var x4132 [1 << 17]byte + var x4133 [1 << 17]byte + var x4134 [1 << 17]byte + var x4135 [1 << 17]byte + var x4136 [1 << 17]byte + var x4137 [1 << 17]byte + var x4138 [1 << 17]byte + var x4139 [1 << 17]byte + var x4140 [1 << 17]byte + var x4141 [1 << 17]byte + var x4142 [1 << 17]byte + var x4143 [1 << 17]byte + var x4144 [1 << 17]byte + var x4145 [1 << 17]byte + var x4146 [1 << 17]byte + var x4147 [1 << 17]byte + var x4148 [1 << 17]byte + var x4149 [1 << 17]byte + var x4150 [1 << 17]byte + var x4151 [1 << 17]byte + var x4152 [1 << 17]byte + var x4153 [1 << 17]byte + var x4154 [1 << 17]byte + var x4155 [1 << 17]byte + var x4156 [1 << 17]byte + var x4157 [1 << 17]byte + var x4158 [1 << 17]byte + var x4159 [1 << 17]byte + var x4160 [1 << 17]byte + var x4161 [1 << 17]byte + var x4162 [1 << 17]byte + var x4163 [1 << 17]byte + var x4164 [1 << 17]byte + var x4165 [1 << 17]byte + var x4166 [1 << 17]byte + var x4167 [1 << 17]byte + var x4168 [1 << 17]byte + var x4169 [1 << 17]byte + var x4170 [1 << 17]byte + var x4171 [1 << 17]byte + var x4172 [1 << 17]byte + var x4173 [1 << 17]byte + var x4174 [1 << 17]byte + var x4175 [1 << 17]byte + var x4176 [1 << 17]byte + var x4177 [1 << 17]byte + var x4178 [1 << 17]byte + var x4179 [1 << 17]byte + var x4180 [1 << 17]byte + var x4181 [1 << 17]byte + var x4182 [1 << 17]byte + var x4183 [1 << 17]byte + var x4184 [1 << 17]byte + var x4185 [1 << 17]byte + var x4186 [1 << 17]byte + var x4187 [1 << 17]byte + var x4188 [1 << 17]byte + var x4189 [1 << 17]byte + var x4190 [1 << 17]byte + var x4191 [1 << 17]byte + var x4192 [1 << 17]byte + var x4193 [1 << 17]byte + var x4194 [1 << 17]byte + var x4195 [1 << 17]byte + var x4196 [1 << 17]byte + var x4197 [1 << 17]byte + var x4198 [1 << 17]byte + var x4199 [1 << 17]byte + var x4200 [1 << 17]byte + var x4201 [1 << 17]byte + var x4202 [1 << 17]byte + var x4203 [1 << 17]byte + var x4204 [1 << 17]byte + var x4205 [1 << 17]byte + var x4206 [1 << 17]byte + var x4207 [1 << 17]byte + var x4208 [1 << 17]byte + var x4209 [1 << 17]byte + var x4210 [1 << 17]byte + var x4211 [1 << 17]byte + var x4212 [1 << 17]byte + var x4213 [1 << 17]byte + var x4214 [1 << 17]byte + var x4215 [1 << 17]byte + var x4216 [1 << 17]byte + var x4217 [1 << 17]byte + var x4218 [1 << 17]byte + var x4219 [1 << 17]byte + var x4220 [1 << 17]byte + var x4221 [1 << 17]byte + var x4222 [1 << 17]byte + var x4223 [1 << 17]byte + var x4224 [1 << 17]byte + var x4225 [1 << 17]byte + var x4226 [1 << 17]byte + var x4227 [1 << 17]byte + var x4228 [1 << 17]byte + var x4229 [1 << 17]byte + var x4230 [1 << 17]byte + var x4231 [1 << 17]byte + var x4232 [1 << 17]byte + var x4233 [1 << 17]byte + var x4234 [1 << 17]byte + var x4235 [1 << 17]byte + var x4236 [1 << 17]byte + var x4237 [1 << 17]byte + var x4238 [1 << 17]byte + var x4239 [1 << 17]byte + var x4240 [1 << 17]byte + var x4241 [1 << 17]byte + var x4242 [1 << 17]byte + var x4243 [1 << 17]byte + var x4244 [1 << 17]byte + var x4245 [1 << 17]byte + var x4246 [1 << 17]byte + var x4247 [1 << 17]byte + var x4248 [1 << 17]byte + var x4249 [1 << 17]byte + var x4250 [1 << 17]byte + var x4251 [1 << 17]byte + var x4252 [1 << 17]byte + var x4253 [1 << 17]byte + var x4254 [1 << 17]byte + var x4255 [1 << 17]byte + var x4256 [1 << 17]byte + var x4257 [1 << 17]byte + var x4258 [1 << 17]byte + var x4259 [1 << 17]byte + var x4260 [1 << 17]byte + var x4261 [1 << 17]byte + var x4262 [1 << 17]byte + var x4263 [1 << 17]byte + var x4264 [1 << 17]byte + var x4265 [1 << 17]byte + var x4266 [1 << 17]byte + var x4267 [1 << 17]byte + var x4268 [1 << 17]byte + var x4269 [1 << 17]byte + var x4270 [1 << 17]byte + var x4271 [1 << 17]byte + var x4272 [1 << 17]byte + var x4273 [1 << 17]byte + var x4274 [1 << 17]byte + var x4275 [1 << 17]byte + var x4276 [1 << 17]byte + var x4277 [1 << 17]byte + var x4278 [1 << 17]byte + var x4279 [1 << 17]byte + var x4280 [1 << 17]byte + var x4281 [1 << 17]byte + var x4282 [1 << 17]byte + var x4283 [1 << 17]byte + var x4284 [1 << 17]byte + var x4285 [1 << 17]byte + var x4286 [1 << 17]byte + var x4287 [1 << 17]byte + var x4288 [1 << 17]byte + var x4289 [1 << 17]byte + var x4290 [1 << 17]byte + var x4291 [1 << 17]byte + var x4292 [1 << 17]byte + var x4293 [1 << 17]byte + var x4294 [1 << 17]byte + var x4295 [1 << 17]byte + var x4296 [1 << 17]byte + var x4297 [1 << 17]byte + var x4298 [1 << 17]byte + var x4299 [1 << 17]byte + var x4300 [1 << 17]byte + var x4301 [1 << 17]byte + var x4302 [1 << 17]byte + var x4303 [1 << 17]byte + var x4304 [1 << 17]byte + var x4305 [1 << 17]byte + var x4306 [1 << 17]byte + var x4307 [1 << 17]byte + var x4308 [1 << 17]byte + var x4309 [1 << 17]byte + var x4310 [1 << 17]byte + var x4311 [1 << 17]byte + var x4312 [1 << 17]byte + var x4313 [1 << 17]byte + var x4314 [1 << 17]byte + var x4315 [1 << 17]byte + var x4316 [1 << 17]byte + var x4317 [1 << 17]byte + var x4318 [1 << 17]byte + var x4319 [1 << 17]byte + var x4320 [1 << 17]byte + var x4321 [1 << 17]byte + var x4322 [1 << 17]byte + var x4323 [1 << 17]byte + var x4324 [1 << 17]byte + var x4325 [1 << 17]byte + var x4326 [1 << 17]byte + var x4327 [1 << 17]byte + var x4328 [1 << 17]byte + var x4329 [1 << 17]byte + var x4330 [1 << 17]byte + var x4331 [1 << 17]byte + var x4332 [1 << 17]byte + var x4333 [1 << 17]byte + var x4334 [1 << 17]byte + var x4335 [1 << 17]byte + var x4336 [1 << 17]byte + var x4337 [1 << 17]byte + var x4338 [1 << 17]byte + var x4339 [1 << 17]byte + var x4340 [1 << 17]byte + var x4341 [1 << 17]byte + var x4342 [1 << 17]byte + var x4343 [1 << 17]byte + var x4344 [1 << 17]byte + var x4345 [1 << 17]byte + var x4346 [1 << 17]byte + var x4347 [1 << 17]byte + var x4348 [1 << 17]byte + var x4349 [1 << 17]byte + var x4350 [1 << 17]byte + var x4351 [1 << 17]byte + var x4352 [1 << 17]byte + var x4353 [1 << 17]byte + var x4354 [1 << 17]byte + var x4355 [1 << 17]byte + var x4356 [1 << 17]byte + var x4357 [1 << 17]byte + var x4358 [1 << 17]byte + var x4359 [1 << 17]byte + var x4360 [1 << 17]byte + var x4361 [1 << 17]byte + var x4362 [1 << 17]byte + var x4363 [1 << 17]byte + var x4364 [1 << 17]byte + var x4365 [1 << 17]byte + var x4366 [1 << 17]byte + var x4367 [1 << 17]byte + var x4368 [1 << 17]byte + var x4369 [1 << 17]byte + var x4370 [1 << 17]byte + var x4371 [1 << 17]byte + var x4372 [1 << 17]byte + var x4373 [1 << 17]byte + var x4374 [1 << 17]byte + var x4375 [1 << 17]byte + var x4376 [1 << 17]byte + var x4377 [1 << 17]byte + var x4378 [1 << 17]byte + var x4379 [1 << 17]byte + var x4380 [1 << 17]byte + var x4381 [1 << 17]byte + var x4382 [1 << 17]byte + var x4383 [1 << 17]byte + var x4384 [1 << 17]byte + var x4385 [1 << 17]byte + var x4386 [1 << 17]byte + var x4387 [1 << 17]byte + var x4388 [1 << 17]byte + var x4389 [1 << 17]byte + var x4390 [1 << 17]byte + var x4391 [1 << 17]byte + var x4392 [1 << 17]byte + var x4393 [1 << 17]byte + var x4394 [1 << 17]byte + var x4395 [1 << 17]byte + var x4396 [1 << 17]byte + var x4397 [1 << 17]byte + var x4398 [1 << 17]byte + var x4399 [1 << 17]byte + var x4400 [1 << 17]byte + var x4401 [1 << 17]byte + var x4402 [1 << 17]byte + var x4403 [1 << 17]byte + var x4404 [1 << 17]byte + var x4405 [1 << 17]byte + var x4406 [1 << 17]byte + var x4407 [1 << 17]byte + var x4408 [1 << 17]byte + var x4409 [1 << 17]byte + var x4410 [1 << 17]byte + var x4411 [1 << 17]byte + var x4412 [1 << 17]byte + var x4413 [1 << 17]byte + var x4414 [1 << 17]byte + var x4415 [1 << 17]byte + var x4416 [1 << 17]byte + var x4417 [1 << 17]byte + var x4418 [1 << 17]byte + var x4419 [1 << 17]byte + var x4420 [1 << 17]byte + var x4421 [1 << 17]byte + var x4422 [1 << 17]byte + var x4423 [1 << 17]byte + var x4424 [1 << 17]byte + var x4425 [1 << 17]byte + var x4426 [1 << 17]byte + var x4427 [1 << 17]byte + var x4428 [1 << 17]byte + var x4429 [1 << 17]byte + var x4430 [1 << 17]byte + var x4431 [1 << 17]byte + var x4432 [1 << 17]byte + var x4433 [1 << 17]byte + var x4434 [1 << 17]byte + var x4435 [1 << 17]byte + var x4436 [1 << 17]byte + var x4437 [1 << 17]byte + var x4438 [1 << 17]byte + var x4439 [1 << 17]byte + var x4440 [1 << 17]byte + var x4441 [1 << 17]byte + var x4442 [1 << 17]byte + var x4443 [1 << 17]byte + var x4444 [1 << 17]byte + var x4445 [1 << 17]byte + var x4446 [1 << 17]byte + var x4447 [1 << 17]byte + var x4448 [1 << 17]byte + var x4449 [1 << 17]byte + var x4450 [1 << 17]byte + var x4451 [1 << 17]byte + var x4452 [1 << 17]byte + var x4453 [1 << 17]byte + var x4454 [1 << 17]byte + var x4455 [1 << 17]byte + var x4456 [1 << 17]byte + var x4457 [1 << 17]byte + var x4458 [1 << 17]byte + var x4459 [1 << 17]byte + var x4460 [1 << 17]byte + var x4461 [1 << 17]byte + var x4462 [1 << 17]byte + var x4463 [1 << 17]byte + var x4464 [1 << 17]byte + var x4465 [1 << 17]byte + var x4466 [1 << 17]byte + var x4467 [1 << 17]byte + var x4468 [1 << 17]byte + var x4469 [1 << 17]byte + var x4470 [1 << 17]byte + var x4471 [1 << 17]byte + var x4472 [1 << 17]byte + var x4473 [1 << 17]byte + var x4474 [1 << 17]byte + var x4475 [1 << 17]byte + var x4476 [1 << 17]byte + var x4477 [1 << 17]byte + var x4478 [1 << 17]byte + var x4479 [1 << 17]byte + var x4480 [1 << 17]byte + var x4481 [1 << 17]byte + var x4482 [1 << 17]byte + var x4483 [1 << 17]byte + var x4484 [1 << 17]byte + var x4485 [1 << 17]byte + var x4486 [1 << 17]byte + var x4487 [1 << 17]byte + var x4488 [1 << 17]byte + var x4489 [1 << 17]byte + var x4490 [1 << 17]byte + var x4491 [1 << 17]byte + var x4492 [1 << 17]byte + var x4493 [1 << 17]byte + var x4494 [1 << 17]byte + var x4495 [1 << 17]byte + var x4496 [1 << 17]byte + var x4497 [1 << 17]byte + var x4498 [1 << 17]byte + var x4499 [1 << 17]byte + var x4500 [1 << 17]byte + var x4501 [1 << 17]byte + var x4502 [1 << 17]byte + var x4503 [1 << 17]byte + var x4504 [1 << 17]byte + var x4505 [1 << 17]byte + var x4506 [1 << 17]byte + var x4507 [1 << 17]byte + var x4508 [1 << 17]byte + var x4509 [1 << 17]byte + var x4510 [1 << 17]byte + var x4511 [1 << 17]byte + var x4512 [1 << 17]byte + var x4513 [1 << 17]byte + var x4514 [1 << 17]byte + var x4515 [1 << 17]byte + var x4516 [1 << 17]byte + var x4517 [1 << 17]byte + var x4518 [1 << 17]byte + var x4519 [1 << 17]byte + var x4520 [1 << 17]byte + var x4521 [1 << 17]byte + var x4522 [1 << 17]byte + var x4523 [1 << 17]byte + var x4524 [1 << 17]byte + var x4525 [1 << 17]byte + var x4526 [1 << 17]byte + var x4527 [1 << 17]byte + var x4528 [1 << 17]byte + var x4529 [1 << 17]byte + var x4530 [1 << 17]byte + var x4531 [1 << 17]byte + var x4532 [1 << 17]byte + var x4533 [1 << 17]byte + var x4534 [1 << 17]byte + var x4535 [1 << 17]byte + var x4536 [1 << 17]byte + var x4537 [1 << 17]byte + var x4538 [1 << 17]byte + var x4539 [1 << 17]byte + var x4540 [1 << 17]byte + var x4541 [1 << 17]byte + var x4542 [1 << 17]byte + var x4543 [1 << 17]byte + var x4544 [1 << 17]byte + var x4545 [1 << 17]byte + var x4546 [1 << 17]byte + var x4547 [1 << 17]byte + var x4548 [1 << 17]byte + var x4549 [1 << 17]byte + var x4550 [1 << 17]byte + var x4551 [1 << 17]byte + var x4552 [1 << 17]byte + var x4553 [1 << 17]byte + var x4554 [1 << 17]byte + var x4555 [1 << 17]byte + var x4556 [1 << 17]byte + var x4557 [1 << 17]byte + var x4558 [1 << 17]byte + var x4559 [1 << 17]byte + var x4560 [1 << 17]byte + var x4561 [1 << 17]byte + var x4562 [1 << 17]byte + var x4563 [1 << 17]byte + var x4564 [1 << 17]byte + var x4565 [1 << 17]byte + var x4566 [1 << 17]byte + var x4567 [1 << 17]byte + var x4568 [1 << 17]byte + var x4569 [1 << 17]byte + var x4570 [1 << 17]byte + var x4571 [1 << 17]byte + var x4572 [1 << 17]byte + var x4573 [1 << 17]byte + var x4574 [1 << 17]byte + var x4575 [1 << 17]byte + var x4576 [1 << 17]byte + var x4577 [1 << 17]byte + var x4578 [1 << 17]byte + var x4579 [1 << 17]byte + var x4580 [1 << 17]byte + var x4581 [1 << 17]byte + var x4582 [1 << 17]byte + var x4583 [1 << 17]byte + var x4584 [1 << 17]byte + var x4585 [1 << 17]byte + var x4586 [1 << 17]byte + var x4587 [1 << 17]byte + var x4588 [1 << 17]byte + var x4589 [1 << 17]byte + var x4590 [1 << 17]byte + var x4591 [1 << 17]byte + var x4592 [1 << 17]byte + var x4593 [1 << 17]byte + var x4594 [1 << 17]byte + var x4595 [1 << 17]byte + var x4596 [1 << 17]byte + var x4597 [1 << 17]byte + var x4598 [1 << 17]byte + var x4599 [1 << 17]byte + var x4600 [1 << 17]byte + var x4601 [1 << 17]byte + var x4602 [1 << 17]byte + var x4603 [1 << 17]byte + var x4604 [1 << 17]byte + var x4605 [1 << 17]byte + var x4606 [1 << 17]byte + var x4607 [1 << 17]byte + var x4608 [1 << 17]byte + var x4609 [1 << 17]byte + var x4610 [1 << 17]byte + var x4611 [1 << 17]byte + var x4612 [1 << 17]byte + var x4613 [1 << 17]byte + var x4614 [1 << 17]byte + var x4615 [1 << 17]byte + var x4616 [1 << 17]byte + var x4617 [1 << 17]byte + var x4618 [1 << 17]byte + var x4619 [1 << 17]byte + var x4620 [1 << 17]byte + var x4621 [1 << 17]byte + var x4622 [1 << 17]byte + var x4623 [1 << 17]byte + var x4624 [1 << 17]byte + var x4625 [1 << 17]byte + var x4626 [1 << 17]byte + var x4627 [1 << 17]byte + var x4628 [1 << 17]byte + var x4629 [1 << 17]byte + var x4630 [1 << 17]byte + var x4631 [1 << 17]byte + var x4632 [1 << 17]byte + var x4633 [1 << 17]byte + var x4634 [1 << 17]byte + var x4635 [1 << 17]byte + var x4636 [1 << 17]byte + var x4637 [1 << 17]byte + var x4638 [1 << 17]byte + var x4639 [1 << 17]byte + var x4640 [1 << 17]byte + var x4641 [1 << 17]byte + var x4642 [1 << 17]byte + var x4643 [1 << 17]byte + var x4644 [1 << 17]byte + var x4645 [1 << 17]byte + var x4646 [1 << 17]byte + var x4647 [1 << 17]byte + var x4648 [1 << 17]byte + var x4649 [1 << 17]byte + var x4650 [1 << 17]byte + var x4651 [1 << 17]byte + var x4652 [1 << 17]byte + var x4653 [1 << 17]byte + var x4654 [1 << 17]byte + var x4655 [1 << 17]byte + var x4656 [1 << 17]byte + var x4657 [1 << 17]byte + var x4658 [1 << 17]byte + var x4659 [1 << 17]byte + var x4660 [1 << 17]byte + var x4661 [1 << 17]byte + var x4662 [1 << 17]byte + var x4663 [1 << 17]byte + var x4664 [1 << 17]byte + var x4665 [1 << 17]byte + var x4666 [1 << 17]byte + var x4667 [1 << 17]byte + var x4668 [1 << 17]byte + var x4669 [1 << 17]byte + var x4670 [1 << 17]byte + var x4671 [1 << 17]byte + var x4672 [1 << 17]byte + var x4673 [1 << 17]byte + var x4674 [1 << 17]byte + var x4675 [1 << 17]byte + var x4676 [1 << 17]byte + var x4677 [1 << 17]byte + var x4678 [1 << 17]byte + var x4679 [1 << 17]byte + var x4680 [1 << 17]byte + var x4681 [1 << 17]byte + var x4682 [1 << 17]byte + var x4683 [1 << 17]byte + var x4684 [1 << 17]byte + var x4685 [1 << 17]byte + var x4686 [1 << 17]byte + var x4687 [1 << 17]byte + var x4688 [1 << 17]byte + var x4689 [1 << 17]byte + var x4690 [1 << 17]byte + var x4691 [1 << 17]byte + var x4692 [1 << 17]byte + var x4693 [1 << 17]byte + var x4694 [1 << 17]byte + var x4695 [1 << 17]byte + var x4696 [1 << 17]byte + var x4697 [1 << 17]byte + var x4698 [1 << 17]byte + var x4699 [1 << 17]byte + var x4700 [1 << 17]byte + var x4701 [1 << 17]byte + var x4702 [1 << 17]byte + var x4703 [1 << 17]byte + var x4704 [1 << 17]byte + var x4705 [1 << 17]byte + var x4706 [1 << 17]byte + var x4707 [1 << 17]byte + var x4708 [1 << 17]byte + var x4709 [1 << 17]byte + var x4710 [1 << 17]byte + var x4711 [1 << 17]byte + var x4712 [1 << 17]byte + var x4713 [1 << 17]byte + var x4714 [1 << 17]byte + var x4715 [1 << 17]byte + var x4716 [1 << 17]byte + var x4717 [1 << 17]byte + var x4718 [1 << 17]byte + var x4719 [1 << 17]byte + var x4720 [1 << 17]byte + var x4721 [1 << 17]byte + var x4722 [1 << 17]byte + var x4723 [1 << 17]byte + var x4724 [1 << 17]byte + var x4725 [1 << 17]byte + var x4726 [1 << 17]byte + var x4727 [1 << 17]byte + var x4728 [1 << 17]byte + var x4729 [1 << 17]byte + var x4730 [1 << 17]byte + var x4731 [1 << 17]byte + var x4732 [1 << 17]byte + var x4733 [1 << 17]byte + var x4734 [1 << 17]byte + var x4735 [1 << 17]byte + var x4736 [1 << 17]byte + var x4737 [1 << 17]byte + var x4738 [1 << 17]byte + var x4739 [1 << 17]byte + var x4740 [1 << 17]byte + var x4741 [1 << 17]byte + var x4742 [1 << 17]byte + var x4743 [1 << 17]byte + var x4744 [1 << 17]byte + var x4745 [1 << 17]byte + var x4746 [1 << 17]byte + var x4747 [1 << 17]byte + var x4748 [1 << 17]byte + var x4749 [1 << 17]byte + var x4750 [1 << 17]byte + var x4751 [1 << 17]byte + var x4752 [1 << 17]byte + var x4753 [1 << 17]byte + var x4754 [1 << 17]byte + var x4755 [1 << 17]byte + var x4756 [1 << 17]byte + var x4757 [1 << 17]byte + var x4758 [1 << 17]byte + var x4759 [1 << 17]byte + var x4760 [1 << 17]byte + var x4761 [1 << 17]byte + var x4762 [1 << 17]byte + var x4763 [1 << 17]byte + var x4764 [1 << 17]byte + var x4765 [1 << 17]byte + var x4766 [1 << 17]byte + var x4767 [1 << 17]byte + var x4768 [1 << 17]byte + var x4769 [1 << 17]byte + var x4770 [1 << 17]byte + var x4771 [1 << 17]byte + var x4772 [1 << 17]byte + var x4773 [1 << 17]byte + var x4774 [1 << 17]byte + var x4775 [1 << 17]byte + var x4776 [1 << 17]byte + var x4777 [1 << 17]byte + var x4778 [1 << 17]byte + var x4779 [1 << 17]byte + var x4780 [1 << 17]byte + var x4781 [1 << 17]byte + var x4782 [1 << 17]byte + var x4783 [1 << 17]byte + var x4784 [1 << 17]byte + var x4785 [1 << 17]byte + var x4786 [1 << 17]byte + var x4787 [1 << 17]byte + var x4788 [1 << 17]byte + var x4789 [1 << 17]byte + var x4790 [1 << 17]byte + var x4791 [1 << 17]byte + var x4792 [1 << 17]byte + var x4793 [1 << 17]byte + var x4794 [1 << 17]byte + var x4795 [1 << 17]byte + var x4796 [1 << 17]byte + var x4797 [1 << 17]byte + var x4798 [1 << 17]byte + var x4799 [1 << 17]byte + var x4800 [1 << 17]byte + var x4801 [1 << 17]byte + var x4802 [1 << 17]byte + var x4803 [1 << 17]byte + var x4804 [1 << 17]byte + var x4805 [1 << 17]byte + var x4806 [1 << 17]byte + var x4807 [1 << 17]byte + var x4808 [1 << 17]byte + var x4809 [1 << 17]byte + var x4810 [1 << 17]byte + var x4811 [1 << 17]byte + var x4812 [1 << 17]byte + var x4813 [1 << 17]byte + var x4814 [1 << 17]byte + var x4815 [1 << 17]byte + var x4816 [1 << 17]byte + var x4817 [1 << 17]byte + var x4818 [1 << 17]byte + var x4819 [1 << 17]byte + var x4820 [1 << 17]byte + var x4821 [1 << 17]byte + var x4822 [1 << 17]byte + var x4823 [1 << 17]byte + var x4824 [1 << 17]byte + var x4825 [1 << 17]byte + var x4826 [1 << 17]byte + var x4827 [1 << 17]byte + var x4828 [1 << 17]byte + var x4829 [1 << 17]byte + var x4830 [1 << 17]byte + var x4831 [1 << 17]byte + var x4832 [1 << 17]byte + var x4833 [1 << 17]byte + var x4834 [1 << 17]byte + var x4835 [1 << 17]byte + var x4836 [1 << 17]byte + var x4837 [1 << 17]byte + var x4838 [1 << 17]byte + var x4839 [1 << 17]byte + var x4840 [1 << 17]byte + var x4841 [1 << 17]byte + var x4842 [1 << 17]byte + var x4843 [1 << 17]byte + var x4844 [1 << 17]byte + var x4845 [1 << 17]byte + var x4846 [1 << 17]byte + var x4847 [1 << 17]byte + var x4848 [1 << 17]byte + var x4849 [1 << 17]byte + var x4850 [1 << 17]byte + var x4851 [1 << 17]byte + var x4852 [1 << 17]byte + var x4853 [1 << 17]byte + var x4854 [1 << 17]byte + var x4855 [1 << 17]byte + var x4856 [1 << 17]byte + var x4857 [1 << 17]byte + var x4858 [1 << 17]byte + var x4859 [1 << 17]byte + var x4860 [1 << 17]byte + var x4861 [1 << 17]byte + var x4862 [1 << 17]byte + var x4863 [1 << 17]byte + var x4864 [1 << 17]byte + var x4865 [1 << 17]byte + var x4866 [1 << 17]byte + var x4867 [1 << 17]byte + var x4868 [1 << 17]byte + var x4869 [1 << 17]byte + var x4870 [1 << 17]byte + var x4871 [1 << 17]byte + var x4872 [1 << 17]byte + var x4873 [1 << 17]byte + var x4874 [1 << 17]byte + var x4875 [1 << 17]byte + var x4876 [1 << 17]byte + var x4877 [1 << 17]byte + var x4878 [1 << 17]byte + var x4879 [1 << 17]byte + var x4880 [1 << 17]byte + var x4881 [1 << 17]byte + var x4882 [1 << 17]byte + var x4883 [1 << 17]byte + var x4884 [1 << 17]byte + var x4885 [1 << 17]byte + var x4886 [1 << 17]byte + var x4887 [1 << 17]byte + var x4888 [1 << 17]byte + var x4889 [1 << 17]byte + var x4890 [1 << 17]byte + var x4891 [1 << 17]byte + var x4892 [1 << 17]byte + var x4893 [1 << 17]byte + var x4894 [1 << 17]byte + var x4895 [1 << 17]byte + var x4896 [1 << 17]byte + var x4897 [1 << 17]byte + var x4898 [1 << 17]byte + var x4899 [1 << 17]byte + var x4900 [1 << 17]byte + var x4901 [1 << 17]byte + var x4902 [1 << 17]byte + var x4903 [1 << 17]byte + var x4904 [1 << 17]byte + var x4905 [1 << 17]byte + var x4906 [1 << 17]byte + var x4907 [1 << 17]byte + var x4908 [1 << 17]byte + var x4909 [1 << 17]byte + var x4910 [1 << 17]byte + var x4911 [1 << 17]byte + var x4912 [1 << 17]byte + var x4913 [1 << 17]byte + var x4914 [1 << 17]byte + var x4915 [1 << 17]byte + var x4916 [1 << 17]byte + var x4917 [1 << 17]byte + var x4918 [1 << 17]byte + var x4919 [1 << 17]byte + var x4920 [1 << 17]byte + var x4921 [1 << 17]byte + var x4922 [1 << 17]byte + var x4923 [1 << 17]byte + var x4924 [1 << 17]byte + var x4925 [1 << 17]byte + var x4926 [1 << 17]byte + var x4927 [1 << 17]byte + var x4928 [1 << 17]byte + var x4929 [1 << 17]byte + var x4930 [1 << 17]byte + var x4931 [1 << 17]byte + var x4932 [1 << 17]byte + var x4933 [1 << 17]byte + var x4934 [1 << 17]byte + var x4935 [1 << 17]byte + var x4936 [1 << 17]byte + var x4937 [1 << 17]byte + var x4938 [1 << 17]byte + var x4939 [1 << 17]byte + var x4940 [1 << 17]byte + var x4941 [1 << 17]byte + var x4942 [1 << 17]byte + var x4943 [1 << 17]byte + var x4944 [1 << 17]byte + var x4945 [1 << 17]byte + var x4946 [1 << 17]byte + var x4947 [1 << 17]byte + var x4948 [1 << 17]byte + var x4949 [1 << 17]byte + var x4950 [1 << 17]byte + var x4951 [1 << 17]byte + var x4952 [1 << 17]byte + var x4953 [1 << 17]byte + var x4954 [1 << 17]byte + var x4955 [1 << 17]byte + var x4956 [1 << 17]byte + var x4957 [1 << 17]byte + var x4958 [1 << 17]byte + var x4959 [1 << 17]byte + var x4960 [1 << 17]byte + var x4961 [1 << 17]byte + var x4962 [1 << 17]byte + var x4963 [1 << 17]byte + var x4964 [1 << 17]byte + var x4965 [1 << 17]byte + var x4966 [1 << 17]byte + var x4967 [1 << 17]byte + var x4968 [1 << 17]byte + var x4969 [1 << 17]byte + var x4970 [1 << 17]byte + var x4971 [1 << 17]byte + var x4972 [1 << 17]byte + var x4973 [1 << 17]byte + var x4974 [1 << 17]byte + var x4975 [1 << 17]byte + var x4976 [1 << 17]byte + var x4977 [1 << 17]byte + var x4978 [1 << 17]byte + var x4979 [1 << 17]byte + var x4980 [1 << 17]byte + var x4981 [1 << 17]byte + var x4982 [1 << 17]byte + var x4983 [1 << 17]byte + var x4984 [1 << 17]byte + var x4985 [1 << 17]byte + var x4986 [1 << 17]byte + var x4987 [1 << 17]byte + var x4988 [1 << 17]byte + var x4989 [1 << 17]byte + var x4990 [1 << 17]byte + var x4991 [1 << 17]byte + var x4992 [1 << 17]byte + var x4993 [1 << 17]byte + var x4994 [1 << 17]byte + var x4995 [1 << 17]byte + var x4996 [1 << 17]byte + var x4997 [1 << 17]byte + var x4998 [1 << 17]byte + var x4999 [1 << 17]byte + var x5000 [1 << 17]byte + var x5001 [1 << 17]byte + var x5002 [1 << 17]byte + var x5003 [1 << 17]byte + var x5004 [1 << 17]byte + var x5005 [1 << 17]byte + var x5006 [1 << 17]byte + var x5007 [1 << 17]byte + var x5008 [1 << 17]byte + var x5009 [1 << 17]byte + var x5010 [1 << 17]byte + var x5011 [1 << 17]byte + var x5012 [1 << 17]byte + var x5013 [1 << 17]byte + var x5014 [1 << 17]byte + var x5015 [1 << 17]byte + var x5016 [1 << 17]byte + var x5017 [1 << 17]byte + var x5018 [1 << 17]byte + var x5019 [1 << 17]byte + var x5020 [1 << 17]byte + var x5021 [1 << 17]byte + var x5022 [1 << 17]byte + var x5023 [1 << 17]byte + var x5024 [1 << 17]byte + var x5025 [1 << 17]byte + var x5026 [1 << 17]byte + var x5027 [1 << 17]byte + var x5028 [1 << 17]byte + var x5029 [1 << 17]byte + var x5030 [1 << 17]byte + var x5031 [1 << 17]byte + var x5032 [1 << 17]byte + var x5033 [1 << 17]byte + var x5034 [1 << 17]byte + var x5035 [1 << 17]byte + var x5036 [1 << 17]byte + var x5037 [1 << 17]byte + var x5038 [1 << 17]byte + var x5039 [1 << 17]byte + var x5040 [1 << 17]byte + var x5041 [1 << 17]byte + var x5042 [1 << 17]byte + var x5043 [1 << 17]byte + var x5044 [1 << 17]byte + var x5045 [1 << 17]byte + var x5046 [1 << 17]byte + var x5047 [1 << 17]byte + var x5048 [1 << 17]byte + var x5049 [1 << 17]byte + var x5050 [1 << 17]byte + var x5051 [1 << 17]byte + var x5052 [1 << 17]byte + var x5053 [1 << 17]byte + var x5054 [1 << 17]byte + var x5055 [1 << 17]byte + var x5056 [1 << 17]byte + var x5057 [1 << 17]byte + var x5058 [1 << 17]byte + var x5059 [1 << 17]byte + var x5060 [1 << 17]byte + var x5061 [1 << 17]byte + var x5062 [1 << 17]byte + var x5063 [1 << 17]byte + var x5064 [1 << 17]byte + var x5065 [1 << 17]byte + var x5066 [1 << 17]byte + var x5067 [1 << 17]byte + var x5068 [1 << 17]byte + var x5069 [1 << 17]byte + var x5070 [1 << 17]byte + var x5071 [1 << 17]byte + var x5072 [1 << 17]byte + var x5073 [1 << 17]byte + var x5074 [1 << 17]byte + var x5075 [1 << 17]byte + var x5076 [1 << 17]byte + var x5077 [1 << 17]byte + var x5078 [1 << 17]byte + var x5079 [1 << 17]byte + var x5080 [1 << 17]byte + var x5081 [1 << 17]byte + var x5082 [1 << 17]byte + var x5083 [1 << 17]byte + var x5084 [1 << 17]byte + var x5085 [1 << 17]byte + var x5086 [1 << 17]byte + var x5087 [1 << 17]byte + var x5088 [1 << 17]byte + var x5089 [1 << 17]byte + var x5090 [1 << 17]byte + var x5091 [1 << 17]byte + var x5092 [1 << 17]byte + var x5093 [1 << 17]byte + var x5094 [1 << 17]byte + var x5095 [1 << 17]byte + var x5096 [1 << 17]byte + var x5097 [1 << 17]byte + var x5098 [1 << 17]byte + var x5099 [1 << 17]byte + var x5100 [1 << 17]byte + var x5101 [1 << 17]byte + var x5102 [1 << 17]byte + var x5103 [1 << 17]byte + var x5104 [1 << 17]byte + var x5105 [1 << 17]byte + var x5106 [1 << 17]byte + var x5107 [1 << 17]byte + var x5108 [1 << 17]byte + var x5109 [1 << 17]byte + var x5110 [1 << 17]byte + var x5111 [1 << 17]byte + var x5112 [1 << 17]byte + var x5113 [1 << 17]byte + var x5114 [1 << 17]byte + var x5115 [1 << 17]byte + var x5116 [1 << 17]byte + var x5117 [1 << 17]byte + var x5118 [1 << 17]byte + var x5119 [1 << 17]byte + var x5120 [1 << 17]byte + var x5121 [1 << 17]byte + var x5122 [1 << 17]byte + var x5123 [1 << 17]byte + var x5124 [1 << 17]byte + var x5125 [1 << 17]byte + var x5126 [1 << 17]byte + var x5127 [1 << 17]byte + var x5128 [1 << 17]byte + var x5129 [1 << 17]byte + var x5130 [1 << 17]byte + var x5131 [1 << 17]byte + var x5132 [1 << 17]byte + var x5133 [1 << 17]byte + var x5134 [1 << 17]byte + var x5135 [1 << 17]byte + var x5136 [1 << 17]byte + var x5137 [1 << 17]byte + var x5138 [1 << 17]byte + var x5139 [1 << 17]byte + var x5140 [1 << 17]byte + var x5141 [1 << 17]byte + var x5142 [1 << 17]byte + var x5143 [1 << 17]byte + var x5144 [1 << 17]byte + var x5145 [1 << 17]byte + var x5146 [1 << 17]byte + var x5147 [1 << 17]byte + var x5148 [1 << 17]byte + var x5149 [1 << 17]byte + var x5150 [1 << 17]byte + var x5151 [1 << 17]byte + var x5152 [1 << 17]byte + var x5153 [1 << 17]byte + var x5154 [1 << 17]byte + var x5155 [1 << 17]byte + var x5156 [1 << 17]byte + var x5157 [1 << 17]byte + var x5158 [1 << 17]byte + var x5159 [1 << 17]byte + var x5160 [1 << 17]byte + var x5161 [1 << 17]byte + var x5162 [1 << 17]byte + var x5163 [1 << 17]byte + var x5164 [1 << 17]byte + var x5165 [1 << 17]byte + var x5166 [1 << 17]byte + var x5167 [1 << 17]byte + var x5168 [1 << 17]byte + var x5169 [1 << 17]byte + var x5170 [1 << 17]byte + var x5171 [1 << 17]byte + var x5172 [1 << 17]byte + var x5173 [1 << 17]byte + var x5174 [1 << 17]byte + var x5175 [1 << 17]byte + var x5176 [1 << 17]byte + var x5177 [1 << 17]byte + var x5178 [1 << 17]byte + var x5179 [1 << 17]byte + var x5180 [1 << 17]byte + var x5181 [1 << 17]byte + var x5182 [1 << 17]byte + var x5183 [1 << 17]byte + var x5184 [1 << 17]byte + var x5185 [1 << 17]byte + var x5186 [1 << 17]byte + var x5187 [1 << 17]byte + var x5188 [1 << 17]byte + var x5189 [1 << 17]byte + var x5190 [1 << 17]byte + var x5191 [1 << 17]byte + var x5192 [1 << 17]byte + var x5193 [1 << 17]byte + var x5194 [1 << 17]byte + var x5195 [1 << 17]byte + var x5196 [1 << 17]byte + var x5197 [1 << 17]byte + var x5198 [1 << 17]byte + var x5199 [1 << 17]byte + var x5200 [1 << 17]byte + var x5201 [1 << 17]byte + var x5202 [1 << 17]byte + var x5203 [1 << 17]byte + var x5204 [1 << 17]byte + var x5205 [1 << 17]byte + var x5206 [1 << 17]byte + var x5207 [1 << 17]byte + var x5208 [1 << 17]byte + var x5209 [1 << 17]byte + var x5210 [1 << 17]byte + var x5211 [1 << 17]byte + var x5212 [1 << 17]byte + var x5213 [1 << 17]byte + var x5214 [1 << 17]byte + var x5215 [1 << 17]byte + var x5216 [1 << 17]byte + var x5217 [1 << 17]byte + var x5218 [1 << 17]byte + var x5219 [1 << 17]byte + var x5220 [1 << 17]byte + var x5221 [1 << 17]byte + var x5222 [1 << 17]byte + var x5223 [1 << 17]byte + var x5224 [1 << 17]byte + var x5225 [1 << 17]byte + var x5226 [1 << 17]byte + var x5227 [1 << 17]byte + var x5228 [1 << 17]byte + var x5229 [1 << 17]byte + var x5230 [1 << 17]byte + var x5231 [1 << 17]byte + var x5232 [1 << 17]byte + var x5233 [1 << 17]byte + var x5234 [1 << 17]byte + var x5235 [1 << 17]byte + var x5236 [1 << 17]byte + var x5237 [1 << 17]byte + var x5238 [1 << 17]byte + var x5239 [1 << 17]byte + var x5240 [1 << 17]byte + var x5241 [1 << 17]byte + var x5242 [1 << 17]byte + var x5243 [1 << 17]byte + var x5244 [1 << 17]byte + var x5245 [1 << 17]byte + var x5246 [1 << 17]byte + var x5247 [1 << 17]byte + var x5248 [1 << 17]byte + var x5249 [1 << 17]byte + var x5250 [1 << 17]byte + var x5251 [1 << 17]byte + var x5252 [1 << 17]byte + var x5253 [1 << 17]byte + var x5254 [1 << 17]byte + var x5255 [1 << 17]byte + var x5256 [1 << 17]byte + var x5257 [1 << 17]byte + var x5258 [1 << 17]byte + var x5259 [1 << 17]byte + var x5260 [1 << 17]byte + var x5261 [1 << 17]byte + var x5262 [1 << 17]byte + var x5263 [1 << 17]byte + var x5264 [1 << 17]byte + var x5265 [1 << 17]byte + var x5266 [1 << 17]byte + var x5267 [1 << 17]byte + var x5268 [1 << 17]byte + var x5269 [1 << 17]byte + var x5270 [1 << 17]byte + var x5271 [1 << 17]byte + var x5272 [1 << 17]byte + var x5273 [1 << 17]byte + var x5274 [1 << 17]byte + var x5275 [1 << 17]byte + var x5276 [1 << 17]byte + var x5277 [1 << 17]byte + var x5278 [1 << 17]byte + var x5279 [1 << 17]byte + var x5280 [1 << 17]byte + var x5281 [1 << 17]byte + var x5282 [1 << 17]byte + var x5283 [1 << 17]byte + var x5284 [1 << 17]byte + var x5285 [1 << 17]byte + var x5286 [1 << 17]byte + var x5287 [1 << 17]byte + var x5288 [1 << 17]byte + var x5289 [1 << 17]byte + var x5290 [1 << 17]byte + var x5291 [1 << 17]byte + var x5292 [1 << 17]byte + var x5293 [1 << 17]byte + var x5294 [1 << 17]byte + var x5295 [1 << 17]byte + var x5296 [1 << 17]byte + var x5297 [1 << 17]byte + var x5298 [1 << 17]byte + var x5299 [1 << 17]byte + var x5300 [1 << 17]byte + var x5301 [1 << 17]byte + var x5302 [1 << 17]byte + var x5303 [1 << 17]byte + var x5304 [1 << 17]byte + var x5305 [1 << 17]byte + var x5306 [1 << 17]byte + var x5307 [1 << 17]byte + var x5308 [1 << 17]byte + var x5309 [1 << 17]byte + var x5310 [1 << 17]byte + var x5311 [1 << 17]byte + var x5312 [1 << 17]byte + var x5313 [1 << 17]byte + var x5314 [1 << 17]byte + var x5315 [1 << 17]byte + var x5316 [1 << 17]byte + var x5317 [1 << 17]byte + var x5318 [1 << 17]byte + var x5319 [1 << 17]byte + var x5320 [1 << 17]byte + var x5321 [1 << 17]byte + var x5322 [1 << 17]byte + var x5323 [1 << 17]byte + var x5324 [1 << 17]byte + var x5325 [1 << 17]byte + var x5326 [1 << 17]byte + var x5327 [1 << 17]byte + var x5328 [1 << 17]byte + var x5329 [1 << 17]byte + var x5330 [1 << 17]byte + var x5331 [1 << 17]byte + var x5332 [1 << 17]byte + var x5333 [1 << 17]byte + var x5334 [1 << 17]byte + var x5335 [1 << 17]byte + var x5336 [1 << 17]byte + var x5337 [1 << 17]byte + var x5338 [1 << 17]byte + var x5339 [1 << 17]byte + var x5340 [1 << 17]byte + var x5341 [1 << 17]byte + var x5342 [1 << 17]byte + var x5343 [1 << 17]byte + var x5344 [1 << 17]byte + var x5345 [1 << 17]byte + var x5346 [1 << 17]byte + var x5347 [1 << 17]byte + var x5348 [1 << 17]byte + var x5349 [1 << 17]byte + var x5350 [1 << 17]byte + var x5351 [1 << 17]byte + var x5352 [1 << 17]byte + var x5353 [1 << 17]byte + var x5354 [1 << 17]byte + var x5355 [1 << 17]byte + var x5356 [1 << 17]byte + var x5357 [1 << 17]byte + var x5358 [1 << 17]byte + var x5359 [1 << 17]byte + var x5360 [1 << 17]byte + var x5361 [1 << 17]byte + var x5362 [1 << 17]byte + var x5363 [1 << 17]byte + var x5364 [1 << 17]byte + var x5365 [1 << 17]byte + var x5366 [1 << 17]byte + var x5367 [1 << 17]byte + var x5368 [1 << 17]byte + var x5369 [1 << 17]byte + var x5370 [1 << 17]byte + var x5371 [1 << 17]byte + var x5372 [1 << 17]byte + var x5373 [1 << 17]byte + var x5374 [1 << 17]byte + var x5375 [1 << 17]byte + var x5376 [1 << 17]byte + var x5377 [1 << 17]byte + var x5378 [1 << 17]byte + var x5379 [1 << 17]byte + var x5380 [1 << 17]byte + var x5381 [1 << 17]byte + var x5382 [1 << 17]byte + var x5383 [1 << 17]byte + var x5384 [1 << 17]byte + var x5385 [1 << 17]byte + var x5386 [1 << 17]byte + var x5387 [1 << 17]byte + var x5388 [1 << 17]byte + var x5389 [1 << 17]byte + var x5390 [1 << 17]byte + var x5391 [1 << 17]byte + var x5392 [1 << 17]byte + var x5393 [1 << 17]byte + var x5394 [1 << 17]byte + var x5395 [1 << 17]byte + var x5396 [1 << 17]byte + var x5397 [1 << 17]byte + var x5398 [1 << 17]byte + var x5399 [1 << 17]byte + var x5400 [1 << 17]byte + var x5401 [1 << 17]byte + var x5402 [1 << 17]byte + var x5403 [1 << 17]byte + var x5404 [1 << 17]byte + var x5405 [1 << 17]byte + var x5406 [1 << 17]byte + var x5407 [1 << 17]byte + var x5408 [1 << 17]byte + var x5409 [1 << 17]byte + var x5410 [1 << 17]byte + var x5411 [1 << 17]byte + var x5412 [1 << 17]byte + var x5413 [1 << 17]byte + var x5414 [1 << 17]byte + var x5415 [1 << 17]byte + var x5416 [1 << 17]byte + var x5417 [1 << 17]byte + var x5418 [1 << 17]byte + var x5419 [1 << 17]byte + var x5420 [1 << 17]byte + var x5421 [1 << 17]byte + var x5422 [1 << 17]byte + var x5423 [1 << 17]byte + var x5424 [1 << 17]byte + var x5425 [1 << 17]byte + var x5426 [1 << 17]byte + var x5427 [1 << 17]byte + var x5428 [1 << 17]byte + var x5429 [1 << 17]byte + var x5430 [1 << 17]byte + var x5431 [1 << 17]byte + var x5432 [1 << 17]byte + var x5433 [1 << 17]byte + var x5434 [1 << 17]byte + var x5435 [1 << 17]byte + var x5436 [1 << 17]byte + var x5437 [1 << 17]byte + var x5438 [1 << 17]byte + var x5439 [1 << 17]byte + var x5440 [1 << 17]byte + var x5441 [1 << 17]byte + var x5442 [1 << 17]byte + var x5443 [1 << 17]byte + var x5444 [1 << 17]byte + var x5445 [1 << 17]byte + var x5446 [1 << 17]byte + var x5447 [1 << 17]byte + var x5448 [1 << 17]byte + var x5449 [1 << 17]byte + var x5450 [1 << 17]byte + var x5451 [1 << 17]byte + var x5452 [1 << 17]byte + var x5453 [1 << 17]byte + var x5454 [1 << 17]byte + var x5455 [1 << 17]byte + var x5456 [1 << 17]byte + var x5457 [1 << 17]byte + var x5458 [1 << 17]byte + var x5459 [1 << 17]byte + var x5460 [1 << 17]byte + var x5461 [1 << 17]byte + var x5462 [1 << 17]byte + var x5463 [1 << 17]byte + var x5464 [1 << 17]byte + var x5465 [1 << 17]byte + var x5466 [1 << 17]byte + var x5467 [1 << 17]byte + var x5468 [1 << 17]byte + var x5469 [1 << 17]byte + var x5470 [1 << 17]byte + var x5471 [1 << 17]byte + var x5472 [1 << 17]byte + var x5473 [1 << 17]byte + var x5474 [1 << 17]byte + var x5475 [1 << 17]byte + var x5476 [1 << 17]byte + var x5477 [1 << 17]byte + var x5478 [1 << 17]byte + var x5479 [1 << 17]byte + var x5480 [1 << 17]byte + var x5481 [1 << 17]byte + var x5482 [1 << 17]byte + var x5483 [1 << 17]byte + var x5484 [1 << 17]byte + var x5485 [1 << 17]byte + var x5486 [1 << 17]byte + var x5487 [1 << 17]byte + var x5488 [1 << 17]byte + var x5489 [1 << 17]byte + var x5490 [1 << 17]byte + var x5491 [1 << 17]byte + var x5492 [1 << 17]byte + var x5493 [1 << 17]byte + var x5494 [1 << 17]byte + var x5495 [1 << 17]byte + var x5496 [1 << 17]byte + var x5497 [1 << 17]byte + var x5498 [1 << 17]byte + var x5499 [1 << 17]byte + var x5500 [1 << 17]byte + var x5501 [1 << 17]byte + var x5502 [1 << 17]byte + var x5503 [1 << 17]byte + var x5504 [1 << 17]byte + var x5505 [1 << 17]byte + var x5506 [1 << 17]byte + var x5507 [1 << 17]byte + var x5508 [1 << 17]byte + var x5509 [1 << 17]byte + var x5510 [1 << 17]byte + var x5511 [1 << 17]byte + var x5512 [1 << 17]byte + var x5513 [1 << 17]byte + var x5514 [1 << 17]byte + var x5515 [1 << 17]byte + var x5516 [1 << 17]byte + var x5517 [1 << 17]byte + var x5518 [1 << 17]byte + var x5519 [1 << 17]byte + var x5520 [1 << 17]byte + var x5521 [1 << 17]byte + var x5522 [1 << 17]byte + var x5523 [1 << 17]byte + var x5524 [1 << 17]byte + var x5525 [1 << 17]byte + var x5526 [1 << 17]byte + var x5527 [1 << 17]byte + var x5528 [1 << 17]byte + var x5529 [1 << 17]byte + var x5530 [1 << 17]byte + var x5531 [1 << 17]byte + var x5532 [1 << 17]byte + var x5533 [1 << 17]byte + var x5534 [1 << 17]byte + var x5535 [1 << 17]byte + var x5536 [1 << 17]byte + var x5537 [1 << 17]byte + var x5538 [1 << 17]byte + var x5539 [1 << 17]byte + var x5540 [1 << 17]byte + var x5541 [1 << 17]byte + var x5542 [1 << 17]byte + var x5543 [1 << 17]byte + var x5544 [1 << 17]byte + var x5545 [1 << 17]byte + var x5546 [1 << 17]byte + var x5547 [1 << 17]byte + var x5548 [1 << 17]byte + var x5549 [1 << 17]byte + var x5550 [1 << 17]byte + var x5551 [1 << 17]byte + var x5552 [1 << 17]byte + var x5553 [1 << 17]byte + var x5554 [1 << 17]byte + var x5555 [1 << 17]byte + var x5556 [1 << 17]byte + var x5557 [1 << 17]byte + var x5558 [1 << 17]byte + var x5559 [1 << 17]byte + var x5560 [1 << 17]byte + var x5561 [1 << 17]byte + var x5562 [1 << 17]byte + var x5563 [1 << 17]byte + var x5564 [1 << 17]byte + var x5565 [1 << 17]byte + var x5566 [1 << 17]byte + var x5567 [1 << 17]byte + var x5568 [1 << 17]byte + var x5569 [1 << 17]byte + var x5570 [1 << 17]byte + var x5571 [1 << 17]byte + var x5572 [1 << 17]byte + var x5573 [1 << 17]byte + var x5574 [1 << 17]byte + var x5575 [1 << 17]byte + var x5576 [1 << 17]byte + var x5577 [1 << 17]byte + var x5578 [1 << 17]byte + var x5579 [1 << 17]byte + var x5580 [1 << 17]byte + var x5581 [1 << 17]byte + var x5582 [1 << 17]byte + var x5583 [1 << 17]byte + var x5584 [1 << 17]byte + var x5585 [1 << 17]byte + var x5586 [1 << 17]byte + var x5587 [1 << 17]byte + var x5588 [1 << 17]byte + var x5589 [1 << 17]byte + var x5590 [1 << 17]byte + var x5591 [1 << 17]byte + var x5592 [1 << 17]byte + var x5593 [1 << 17]byte + var x5594 [1 << 17]byte + var x5595 [1 << 17]byte + var x5596 [1 << 17]byte + var x5597 [1 << 17]byte + var x5598 [1 << 17]byte + var x5599 [1 << 17]byte + var x5600 [1 << 17]byte + var x5601 [1 << 17]byte + var x5602 [1 << 17]byte + var x5603 [1 << 17]byte + var x5604 [1 << 17]byte + var x5605 [1 << 17]byte + var x5606 [1 << 17]byte + var x5607 [1 << 17]byte + var x5608 [1 << 17]byte + var x5609 [1 << 17]byte + var x5610 [1 << 17]byte + var x5611 [1 << 17]byte + var x5612 [1 << 17]byte + var x5613 [1 << 17]byte + var x5614 [1 << 17]byte + var x5615 [1 << 17]byte + var x5616 [1 << 17]byte + var x5617 [1 << 17]byte + var x5618 [1 << 17]byte + var x5619 [1 << 17]byte + var x5620 [1 << 17]byte + var x5621 [1 << 17]byte + var x5622 [1 << 17]byte + var x5623 [1 << 17]byte + var x5624 [1 << 17]byte + var x5625 [1 << 17]byte + var x5626 [1 << 17]byte + var x5627 [1 << 17]byte + var x5628 [1 << 17]byte + var x5629 [1 << 17]byte + var x5630 [1 << 17]byte + var x5631 [1 << 17]byte + var x5632 [1 << 17]byte + var x5633 [1 << 17]byte + var x5634 [1 << 17]byte + var x5635 [1 << 17]byte + var x5636 [1 << 17]byte + var x5637 [1 << 17]byte + var x5638 [1 << 17]byte + var x5639 [1 << 17]byte + var x5640 [1 << 17]byte + var x5641 [1 << 17]byte + var x5642 [1 << 17]byte + var x5643 [1 << 17]byte + var x5644 [1 << 17]byte + var x5645 [1 << 17]byte + var x5646 [1 << 17]byte + var x5647 [1 << 17]byte + var x5648 [1 << 17]byte + var x5649 [1 << 17]byte + var x5650 [1 << 17]byte + var x5651 [1 << 17]byte + var x5652 [1 << 17]byte + var x5653 [1 << 17]byte + var x5654 [1 << 17]byte + var x5655 [1 << 17]byte + var x5656 [1 << 17]byte + var x5657 [1 << 17]byte + var x5658 [1 << 17]byte + var x5659 [1 << 17]byte + var x5660 [1 << 17]byte + var x5661 [1 << 17]byte + var x5662 [1 << 17]byte + var x5663 [1 << 17]byte + var x5664 [1 << 17]byte + var x5665 [1 << 17]byte + var x5666 [1 << 17]byte + var x5667 [1 << 17]byte + var x5668 [1 << 17]byte + var x5669 [1 << 17]byte + var x5670 [1 << 17]byte + var x5671 [1 << 17]byte + var x5672 [1 << 17]byte + var x5673 [1 << 17]byte + var x5674 [1 << 17]byte + var x5675 [1 << 17]byte + var x5676 [1 << 17]byte + var x5677 [1 << 17]byte + var x5678 [1 << 17]byte + var x5679 [1 << 17]byte + var x5680 [1 << 17]byte + var x5681 [1 << 17]byte + var x5682 [1 << 17]byte + var x5683 [1 << 17]byte + var x5684 [1 << 17]byte + var x5685 [1 << 17]byte + var x5686 [1 << 17]byte + var x5687 [1 << 17]byte + var x5688 [1 << 17]byte + var x5689 [1 << 17]byte + var x5690 [1 << 17]byte + var x5691 [1 << 17]byte + var x5692 [1 << 17]byte + var x5693 [1 << 17]byte + var x5694 [1 << 17]byte + var x5695 [1 << 17]byte + var x5696 [1 << 17]byte + var x5697 [1 << 17]byte + var x5698 [1 << 17]byte + var x5699 [1 << 17]byte + var x5700 [1 << 17]byte + var x5701 [1 << 17]byte + var x5702 [1 << 17]byte + var x5703 [1 << 17]byte + var x5704 [1 << 17]byte + var x5705 [1 << 17]byte + var x5706 [1 << 17]byte + var x5707 [1 << 17]byte + var x5708 [1 << 17]byte + var x5709 [1 << 17]byte + var x5710 [1 << 17]byte + var x5711 [1 << 17]byte + var x5712 [1 << 17]byte + var x5713 [1 << 17]byte + var x5714 [1 << 17]byte + var x5715 [1 << 17]byte + var x5716 [1 << 17]byte + var x5717 [1 << 17]byte + var x5718 [1 << 17]byte + var x5719 [1 << 17]byte + var x5720 [1 << 17]byte + var x5721 [1 << 17]byte + var x5722 [1 << 17]byte + var x5723 [1 << 17]byte + var x5724 [1 << 17]byte + var x5725 [1 << 17]byte + var x5726 [1 << 17]byte + var x5727 [1 << 17]byte + var x5728 [1 << 17]byte + var x5729 [1 << 17]byte + var x5730 [1 << 17]byte + var x5731 [1 << 17]byte + var x5732 [1 << 17]byte + var x5733 [1 << 17]byte + var x5734 [1 << 17]byte + var x5735 [1 << 17]byte + var x5736 [1 << 17]byte + var x5737 [1 << 17]byte + var x5738 [1 << 17]byte + var x5739 [1 << 17]byte + var x5740 [1 << 17]byte + var x5741 [1 << 17]byte + var x5742 [1 << 17]byte + var x5743 [1 << 17]byte + var x5744 [1 << 17]byte + var x5745 [1 << 17]byte + var x5746 [1 << 17]byte + var x5747 [1 << 17]byte + var x5748 [1 << 17]byte + var x5749 [1 << 17]byte + var x5750 [1 << 17]byte + var x5751 [1 << 17]byte + var x5752 [1 << 17]byte + var x5753 [1 << 17]byte + var x5754 [1 << 17]byte + var x5755 [1 << 17]byte + var x5756 [1 << 17]byte + var x5757 [1 << 17]byte + var x5758 [1 << 17]byte + var x5759 [1 << 17]byte + var x5760 [1 << 17]byte + var x5761 [1 << 17]byte + var x5762 [1 << 17]byte + var x5763 [1 << 17]byte + var x5764 [1 << 17]byte + var x5765 [1 << 17]byte + var x5766 [1 << 17]byte + var x5767 [1 << 17]byte + var x5768 [1 << 17]byte + var x5769 [1 << 17]byte + var x5770 [1 << 17]byte + var x5771 [1 << 17]byte + var x5772 [1 << 17]byte + var x5773 [1 << 17]byte + var x5774 [1 << 17]byte + var x5775 [1 << 17]byte + var x5776 [1 << 17]byte + var x5777 [1 << 17]byte + var x5778 [1 << 17]byte + var x5779 [1 << 17]byte + var x5780 [1 << 17]byte + var x5781 [1 << 17]byte + var x5782 [1 << 17]byte + var x5783 [1 << 17]byte + var x5784 [1 << 17]byte + var x5785 [1 << 17]byte + var x5786 [1 << 17]byte + var x5787 [1 << 17]byte + var x5788 [1 << 17]byte + var x5789 [1 << 17]byte + var x5790 [1 << 17]byte + var x5791 [1 << 17]byte + var x5792 [1 << 17]byte + var x5793 [1 << 17]byte + var x5794 [1 << 17]byte + var x5795 [1 << 17]byte + var x5796 [1 << 17]byte + var x5797 [1 << 17]byte + var x5798 [1 << 17]byte + var x5799 [1 << 17]byte + var x5800 [1 << 17]byte + var x5801 [1 << 17]byte + var x5802 [1 << 17]byte + var x5803 [1 << 17]byte + var x5804 [1 << 17]byte + var x5805 [1 << 17]byte + var x5806 [1 << 17]byte + var x5807 [1 << 17]byte + var x5808 [1 << 17]byte + var x5809 [1 << 17]byte + var x5810 [1 << 17]byte + var x5811 [1 << 17]byte + var x5812 [1 << 17]byte + var x5813 [1 << 17]byte + var x5814 [1 << 17]byte + var x5815 [1 << 17]byte + var x5816 [1 << 17]byte + var x5817 [1 << 17]byte + var x5818 [1 << 17]byte + var x5819 [1 << 17]byte + var x5820 [1 << 17]byte + var x5821 [1 << 17]byte + var x5822 [1 << 17]byte + var x5823 [1 << 17]byte + var x5824 [1 << 17]byte + var x5825 [1 << 17]byte + var x5826 [1 << 17]byte + var x5827 [1 << 17]byte + var x5828 [1 << 17]byte + var x5829 [1 << 17]byte + var x5830 [1 << 17]byte + var x5831 [1 << 17]byte + var x5832 [1 << 17]byte + var x5833 [1 << 17]byte + var x5834 [1 << 17]byte + var x5835 [1 << 17]byte + var x5836 [1 << 17]byte + var x5837 [1 << 17]byte + var x5838 [1 << 17]byte + var x5839 [1 << 17]byte + var x5840 [1 << 17]byte + var x5841 [1 << 17]byte + var x5842 [1 << 17]byte + var x5843 [1 << 17]byte + var x5844 [1 << 17]byte + var x5845 [1 << 17]byte + var x5846 [1 << 17]byte + var x5847 [1 << 17]byte + var x5848 [1 << 17]byte + var x5849 [1 << 17]byte + var x5850 [1 << 17]byte + var x5851 [1 << 17]byte + var x5852 [1 << 17]byte + var x5853 [1 << 17]byte + var x5854 [1 << 17]byte + var x5855 [1 << 17]byte + var x5856 [1 << 17]byte + var x5857 [1 << 17]byte + var x5858 [1 << 17]byte + var x5859 [1 << 17]byte + var x5860 [1 << 17]byte + var x5861 [1 << 17]byte + var x5862 [1 << 17]byte + var x5863 [1 << 17]byte + var x5864 [1 << 17]byte + var x5865 [1 << 17]byte + var x5866 [1 << 17]byte + var x5867 [1 << 17]byte + var x5868 [1 << 17]byte + var x5869 [1 << 17]byte + var x5870 [1 << 17]byte + var x5871 [1 << 17]byte + var x5872 [1 << 17]byte + var x5873 [1 << 17]byte + var x5874 [1 << 17]byte + var x5875 [1 << 17]byte + var x5876 [1 << 17]byte + var x5877 [1 << 17]byte + var x5878 [1 << 17]byte + var x5879 [1 << 17]byte + var x5880 [1 << 17]byte + var x5881 [1 << 17]byte + var x5882 [1 << 17]byte + var x5883 [1 << 17]byte + var x5884 [1 << 17]byte + var x5885 [1 << 17]byte + var x5886 [1 << 17]byte + var x5887 [1 << 17]byte + var x5888 [1 << 17]byte + var x5889 [1 << 17]byte + var x5890 [1 << 17]byte + var x5891 [1 << 17]byte + var x5892 [1 << 17]byte + var x5893 [1 << 17]byte + var x5894 [1 << 17]byte + var x5895 [1 << 17]byte + var x5896 [1 << 17]byte + var x5897 [1 << 17]byte + var x5898 [1 << 17]byte + var x5899 [1 << 17]byte + var x5900 [1 << 17]byte + var x5901 [1 << 17]byte + var x5902 [1 << 17]byte + var x5903 [1 << 17]byte + var x5904 [1 << 17]byte + var x5905 [1 << 17]byte + var x5906 [1 << 17]byte + var x5907 [1 << 17]byte + var x5908 [1 << 17]byte + var x5909 [1 << 17]byte + var x5910 [1 << 17]byte + var x5911 [1 << 17]byte + var x5912 [1 << 17]byte + var x5913 [1 << 17]byte + var x5914 [1 << 17]byte + var x5915 [1 << 17]byte + var x5916 [1 << 17]byte + var x5917 [1 << 17]byte + var x5918 [1 << 17]byte + var x5919 [1 << 17]byte + var x5920 [1 << 17]byte + var x5921 [1 << 17]byte + var x5922 [1 << 17]byte + var x5923 [1 << 17]byte + var x5924 [1 << 17]byte + var x5925 [1 << 17]byte + var x5926 [1 << 17]byte + var x5927 [1 << 17]byte + var x5928 [1 << 17]byte + var x5929 [1 << 17]byte + var x5930 [1 << 17]byte + var x5931 [1 << 17]byte + var x5932 [1 << 17]byte + var x5933 [1 << 17]byte + var x5934 [1 << 17]byte + var x5935 [1 << 17]byte + var x5936 [1 << 17]byte + var x5937 [1 << 17]byte + var x5938 [1 << 17]byte + var x5939 [1 << 17]byte + var x5940 [1 << 17]byte + var x5941 [1 << 17]byte + var x5942 [1 << 17]byte + var x5943 [1 << 17]byte + var x5944 [1 << 17]byte + var x5945 [1 << 17]byte + var x5946 [1 << 17]byte + var x5947 [1 << 17]byte + var x5948 [1 << 17]byte + var x5949 [1 << 17]byte + var x5950 [1 << 17]byte + var x5951 [1 << 17]byte + var x5952 [1 << 17]byte + var x5953 [1 << 17]byte + var x5954 [1 << 17]byte + var x5955 [1 << 17]byte + var x5956 [1 << 17]byte + var x5957 [1 << 17]byte + var x5958 [1 << 17]byte + var x5959 [1 << 17]byte + var x5960 [1 << 17]byte + var x5961 [1 << 17]byte + var x5962 [1 << 17]byte + var x5963 [1 << 17]byte + var x5964 [1 << 17]byte + var x5965 [1 << 17]byte + var x5966 [1 << 17]byte + var x5967 [1 << 17]byte + var x5968 [1 << 17]byte + var x5969 [1 << 17]byte + var x5970 [1 << 17]byte + var x5971 [1 << 17]byte + var x5972 [1 << 17]byte + var x5973 [1 << 17]byte + var x5974 [1 << 17]byte + var x5975 [1 << 17]byte + var x5976 [1 << 17]byte + var x5977 [1 << 17]byte + var x5978 [1 << 17]byte + var x5979 [1 << 17]byte + var x5980 [1 << 17]byte + var x5981 [1 << 17]byte + var x5982 [1 << 17]byte + var x5983 [1 << 17]byte + var x5984 [1 << 17]byte + var x5985 [1 << 17]byte + var x5986 [1 << 17]byte + var x5987 [1 << 17]byte + var x5988 [1 << 17]byte + var x5989 [1 << 17]byte + var x5990 [1 << 17]byte + var x5991 [1 << 17]byte + var x5992 [1 << 17]byte + var x5993 [1 << 17]byte + var x5994 [1 << 17]byte + var x5995 [1 << 17]byte + var x5996 [1 << 17]byte + var x5997 [1 << 17]byte + var x5998 [1 << 17]byte + var x5999 [1 << 17]byte + var x6000 [1 << 17]byte + var x6001 [1 << 17]byte + var x6002 [1 << 17]byte + var x6003 [1 << 17]byte + var x6004 [1 << 17]byte + var x6005 [1 << 17]byte + var x6006 [1 << 17]byte + var x6007 [1 << 17]byte + var x6008 [1 << 17]byte + var x6009 [1 << 17]byte + var x6010 [1 << 17]byte + var x6011 [1 << 17]byte + var x6012 [1 << 17]byte + var x6013 [1 << 17]byte + var x6014 [1 << 17]byte + var x6015 [1 << 17]byte + var x6016 [1 << 17]byte + var x6017 [1 << 17]byte + var x6018 [1 << 17]byte + var x6019 [1 << 17]byte + var x6020 [1 << 17]byte + var x6021 [1 << 17]byte + var x6022 [1 << 17]byte + var x6023 [1 << 17]byte + var x6024 [1 << 17]byte + var x6025 [1 << 17]byte + var x6026 [1 << 17]byte + var x6027 [1 << 17]byte + var x6028 [1 << 17]byte + var x6029 [1 << 17]byte + var x6030 [1 << 17]byte + var x6031 [1 << 17]byte + var x6032 [1 << 17]byte + var x6033 [1 << 17]byte + var x6034 [1 << 17]byte + var x6035 [1 << 17]byte + var x6036 [1 << 17]byte + var x6037 [1 << 17]byte + var x6038 [1 << 17]byte + var x6039 [1 << 17]byte + var x6040 [1 << 17]byte + var x6041 [1 << 17]byte + var x6042 [1 << 17]byte + var x6043 [1 << 17]byte + var x6044 [1 << 17]byte + var x6045 [1 << 17]byte + var x6046 [1 << 17]byte + var x6047 [1 << 17]byte + var x6048 [1 << 17]byte + var x6049 [1 << 17]byte + var x6050 [1 << 17]byte + var x6051 [1 << 17]byte + var x6052 [1 << 17]byte + var x6053 [1 << 17]byte + var x6054 [1 << 17]byte + var x6055 [1 << 17]byte + var x6056 [1 << 17]byte + var x6057 [1 << 17]byte + var x6058 [1 << 17]byte + var x6059 [1 << 17]byte + var x6060 [1 << 17]byte + var x6061 [1 << 17]byte + var x6062 [1 << 17]byte + var x6063 [1 << 17]byte + var x6064 [1 << 17]byte + var x6065 [1 << 17]byte + var x6066 [1 << 17]byte + var x6067 [1 << 17]byte + var x6068 [1 << 17]byte + var x6069 [1 << 17]byte + var x6070 [1 << 17]byte + var x6071 [1 << 17]byte + var x6072 [1 << 17]byte + var x6073 [1 << 17]byte + var x6074 [1 << 17]byte + var x6075 [1 << 17]byte + var x6076 [1 << 17]byte + var x6077 [1 << 17]byte + var x6078 [1 << 17]byte + var x6079 [1 << 17]byte + var x6080 [1 << 17]byte + var x6081 [1 << 17]byte + var x6082 [1 << 17]byte + var x6083 [1 << 17]byte + var x6084 [1 << 17]byte + var x6085 [1 << 17]byte + var x6086 [1 << 17]byte + var x6087 [1 << 17]byte + var x6088 [1 << 17]byte + var x6089 [1 << 17]byte + var x6090 [1 << 17]byte + var x6091 [1 << 17]byte + var x6092 [1 << 17]byte + var x6093 [1 << 17]byte + var x6094 [1 << 17]byte + var x6095 [1 << 17]byte + var x6096 [1 << 17]byte + var x6097 [1 << 17]byte + var x6098 [1 << 17]byte + var x6099 [1 << 17]byte + var x6100 [1 << 17]byte + var x6101 [1 << 17]byte + var x6102 [1 << 17]byte + var x6103 [1 << 17]byte + var x6104 [1 << 17]byte + var x6105 [1 << 17]byte + var x6106 [1 << 17]byte + var x6107 [1 << 17]byte + var x6108 [1 << 17]byte + var x6109 [1 << 17]byte + var x6110 [1 << 17]byte + var x6111 [1 << 17]byte + var x6112 [1 << 17]byte + var x6113 [1 << 17]byte + var x6114 [1 << 17]byte + var x6115 [1 << 17]byte + var x6116 [1 << 17]byte + var x6117 [1 << 17]byte + var x6118 [1 << 17]byte + var x6119 [1 << 17]byte + var x6120 [1 << 17]byte + var x6121 [1 << 17]byte + var x6122 [1 << 17]byte + var x6123 [1 << 17]byte + var x6124 [1 << 17]byte + var x6125 [1 << 17]byte + var x6126 [1 << 17]byte + var x6127 [1 << 17]byte + var x6128 [1 << 17]byte + var x6129 [1 << 17]byte + var x6130 [1 << 17]byte + var x6131 [1 << 17]byte + var x6132 [1 << 17]byte + var x6133 [1 << 17]byte + var x6134 [1 << 17]byte + var x6135 [1 << 17]byte + var x6136 [1 << 17]byte + var x6137 [1 << 17]byte + var x6138 [1 << 17]byte + var x6139 [1 << 17]byte + var x6140 [1 << 17]byte + var x6141 [1 << 17]byte + var x6142 [1 << 17]byte + var x6143 [1 << 17]byte + var x6144 [1 << 17]byte + var x6145 [1 << 17]byte + var x6146 [1 << 17]byte + var x6147 [1 << 17]byte + var x6148 [1 << 17]byte + var x6149 [1 << 17]byte + var x6150 [1 << 17]byte + var x6151 [1 << 17]byte + var x6152 [1 << 17]byte + var x6153 [1 << 17]byte + var x6154 [1 << 17]byte + var x6155 [1 << 17]byte + var x6156 [1 << 17]byte + var x6157 [1 << 17]byte + var x6158 [1 << 17]byte + var x6159 [1 << 17]byte + var x6160 [1 << 17]byte + var x6161 [1 << 17]byte + var x6162 [1 << 17]byte + var x6163 [1 << 17]byte + var x6164 [1 << 17]byte + var x6165 [1 << 17]byte + var x6166 [1 << 17]byte + var x6167 [1 << 17]byte + var x6168 [1 << 17]byte + var x6169 [1 << 17]byte + var x6170 [1 << 17]byte + var x6171 [1 << 17]byte + var x6172 [1 << 17]byte + var x6173 [1 << 17]byte + var x6174 [1 << 17]byte + var x6175 [1 << 17]byte + var x6176 [1 << 17]byte + var x6177 [1 << 17]byte + var x6178 [1 << 17]byte + var x6179 [1 << 17]byte + var x6180 [1 << 17]byte + var x6181 [1 << 17]byte + var x6182 [1 << 17]byte + var x6183 [1 << 17]byte + var x6184 [1 << 17]byte + var x6185 [1 << 17]byte + var x6186 [1 << 17]byte + var x6187 [1 << 17]byte + var x6188 [1 << 17]byte + var x6189 [1 << 17]byte + var x6190 [1 << 17]byte + var x6191 [1 << 17]byte + var x6192 [1 << 17]byte + var x6193 [1 << 17]byte + var x6194 [1 << 17]byte + var x6195 [1 << 17]byte + var x6196 [1 << 17]byte + var x6197 [1 << 17]byte + var x6198 [1 << 17]byte + var x6199 [1 << 17]byte + var x6200 [1 << 17]byte + var x6201 [1 << 17]byte + var x6202 [1 << 17]byte + var x6203 [1 << 17]byte + var x6204 [1 << 17]byte + var x6205 [1 << 17]byte + var x6206 [1 << 17]byte + var x6207 [1 << 17]byte + var x6208 [1 << 17]byte + var x6209 [1 << 17]byte + var x6210 [1 << 17]byte + var x6211 [1 << 17]byte + var x6212 [1 << 17]byte + var x6213 [1 << 17]byte + var x6214 [1 << 17]byte + var x6215 [1 << 17]byte + var x6216 [1 << 17]byte + var x6217 [1 << 17]byte + var x6218 [1 << 17]byte + var x6219 [1 << 17]byte + var x6220 [1 << 17]byte + var x6221 [1 << 17]byte + var x6222 [1 << 17]byte + var x6223 [1 << 17]byte + var x6224 [1 << 17]byte + var x6225 [1 << 17]byte + var x6226 [1 << 17]byte + var x6227 [1 << 17]byte + var x6228 [1 << 17]byte + var x6229 [1 << 17]byte + var x6230 [1 << 17]byte + var x6231 [1 << 17]byte + var x6232 [1 << 17]byte + var x6233 [1 << 17]byte + var x6234 [1 << 17]byte + var x6235 [1 << 17]byte + var x6236 [1 << 17]byte + var x6237 [1 << 17]byte + var x6238 [1 << 17]byte + var x6239 [1 << 17]byte + var x6240 [1 << 17]byte + var x6241 [1 << 17]byte + var x6242 [1 << 17]byte + var x6243 [1 << 17]byte + var x6244 [1 << 17]byte + var x6245 [1 << 17]byte + var x6246 [1 << 17]byte + var x6247 [1 << 17]byte + var x6248 [1 << 17]byte + var x6249 [1 << 17]byte + var x6250 [1 << 17]byte + var x6251 [1 << 17]byte + var x6252 [1 << 17]byte + var x6253 [1 << 17]byte + var x6254 [1 << 17]byte + var x6255 [1 << 17]byte + var x6256 [1 << 17]byte + var x6257 [1 << 17]byte + var x6258 [1 << 17]byte + var x6259 [1 << 17]byte + var x6260 [1 << 17]byte + var x6261 [1 << 17]byte + var x6262 [1 << 17]byte + var x6263 [1 << 17]byte + var x6264 [1 << 17]byte + var x6265 [1 << 17]byte + var x6266 [1 << 17]byte + var x6267 [1 << 17]byte + var x6268 [1 << 17]byte + var x6269 [1 << 17]byte + var x6270 [1 << 17]byte + var x6271 [1 << 17]byte + var x6272 [1 << 17]byte + var x6273 [1 << 17]byte + var x6274 [1 << 17]byte + var x6275 [1 << 17]byte + var x6276 [1 << 17]byte + var x6277 [1 << 17]byte + var x6278 [1 << 17]byte + var x6279 [1 << 17]byte + var x6280 [1 << 17]byte + var x6281 [1 << 17]byte + var x6282 [1 << 17]byte + var x6283 [1 << 17]byte + var x6284 [1 << 17]byte + var x6285 [1 << 17]byte + var x6286 [1 << 17]byte + var x6287 [1 << 17]byte + var x6288 [1 << 17]byte + var x6289 [1 << 17]byte + var x6290 [1 << 17]byte + var x6291 [1 << 17]byte + var x6292 [1 << 17]byte + var x6293 [1 << 17]byte + var x6294 [1 << 17]byte + var x6295 [1 << 17]byte + var x6296 [1 << 17]byte + var x6297 [1 << 17]byte + var x6298 [1 << 17]byte + var x6299 [1 << 17]byte + var x6300 [1 << 17]byte + var x6301 [1 << 17]byte + var x6302 [1 << 17]byte + var x6303 [1 << 17]byte + var x6304 [1 << 17]byte + var x6305 [1 << 17]byte + var x6306 [1 << 17]byte + var x6307 [1 << 17]byte + var x6308 [1 << 17]byte + var x6309 [1 << 17]byte + var x6310 [1 << 17]byte + var x6311 [1 << 17]byte + var x6312 [1 << 17]byte + var x6313 [1 << 17]byte + var x6314 [1 << 17]byte + var x6315 [1 << 17]byte + var x6316 [1 << 17]byte + var x6317 [1 << 17]byte + var x6318 [1 << 17]byte + var x6319 [1 << 17]byte + var x6320 [1 << 17]byte + var x6321 [1 << 17]byte + var x6322 [1 << 17]byte + var x6323 [1 << 17]byte + var x6324 [1 << 17]byte + var x6325 [1 << 17]byte + var x6326 [1 << 17]byte + var x6327 [1 << 17]byte + var x6328 [1 << 17]byte + var x6329 [1 << 17]byte + var x6330 [1 << 17]byte + var x6331 [1 << 17]byte + var x6332 [1 << 17]byte + var x6333 [1 << 17]byte + var x6334 [1 << 17]byte + var x6335 [1 << 17]byte + var x6336 [1 << 17]byte + var x6337 [1 << 17]byte + var x6338 [1 << 17]byte + var x6339 [1 << 17]byte + var x6340 [1 << 17]byte + var x6341 [1 << 17]byte + var x6342 [1 << 17]byte + var x6343 [1 << 17]byte + var x6344 [1 << 17]byte + var x6345 [1 << 17]byte + var x6346 [1 << 17]byte + var x6347 [1 << 17]byte + var x6348 [1 << 17]byte + var x6349 [1 << 17]byte + var x6350 [1 << 17]byte + var x6351 [1 << 17]byte + var x6352 [1 << 17]byte + var x6353 [1 << 17]byte + var x6354 [1 << 17]byte + var x6355 [1 << 17]byte + var x6356 [1 << 17]byte + var x6357 [1 << 17]byte + var x6358 [1 << 17]byte + var x6359 [1 << 17]byte + var x6360 [1 << 17]byte + var x6361 [1 << 17]byte + var x6362 [1 << 17]byte + var x6363 [1 << 17]byte + var x6364 [1 << 17]byte + var x6365 [1 << 17]byte + var x6366 [1 << 17]byte + var x6367 [1 << 17]byte + var x6368 [1 << 17]byte + var x6369 [1 << 17]byte + var x6370 [1 << 17]byte + var x6371 [1 << 17]byte + var x6372 [1 << 17]byte + var x6373 [1 << 17]byte + var x6374 [1 << 17]byte + var x6375 [1 << 17]byte + var x6376 [1 << 17]byte + var x6377 [1 << 17]byte + var x6378 [1 << 17]byte + var x6379 [1 << 17]byte + var x6380 [1 << 17]byte + var x6381 [1 << 17]byte + var x6382 [1 << 17]byte + var x6383 [1 << 17]byte + var x6384 [1 << 17]byte + var x6385 [1 << 17]byte + var x6386 [1 << 17]byte + var x6387 [1 << 17]byte + var x6388 [1 << 17]byte + var x6389 [1 << 17]byte + var x6390 [1 << 17]byte + var x6391 [1 << 17]byte + var x6392 [1 << 17]byte + var x6393 [1 << 17]byte + var x6394 [1 << 17]byte + var x6395 [1 << 17]byte + var x6396 [1 << 17]byte + var x6397 [1 << 17]byte + var x6398 [1 << 17]byte + var x6399 [1 << 17]byte + var x6400 [1 << 17]byte + var x6401 [1 << 17]byte + var x6402 [1 << 17]byte + var x6403 [1 << 17]byte + var x6404 [1 << 17]byte + var x6405 [1 << 17]byte + var x6406 [1 << 17]byte + var x6407 [1 << 17]byte + var x6408 [1 << 17]byte + var x6409 [1 << 17]byte + var x6410 [1 << 17]byte + var x6411 [1 << 17]byte + var x6412 [1 << 17]byte + var x6413 [1 << 17]byte + var x6414 [1 << 17]byte + var x6415 [1 << 17]byte + var x6416 [1 << 17]byte + var x6417 [1 << 17]byte + var x6418 [1 << 17]byte + var x6419 [1 << 17]byte + var x6420 [1 << 17]byte + var x6421 [1 << 17]byte + var x6422 [1 << 17]byte + var x6423 [1 << 17]byte + var x6424 [1 << 17]byte + var x6425 [1 << 17]byte + var x6426 [1 << 17]byte + var x6427 [1 << 17]byte + var x6428 [1 << 17]byte + var x6429 [1 << 17]byte + var x6430 [1 << 17]byte + var x6431 [1 << 17]byte + var x6432 [1 << 17]byte + var x6433 [1 << 17]byte + var x6434 [1 << 17]byte + var x6435 [1 << 17]byte + var x6436 [1 << 17]byte + var x6437 [1 << 17]byte + var x6438 [1 << 17]byte + var x6439 [1 << 17]byte + var x6440 [1 << 17]byte + var x6441 [1 << 17]byte + var x6442 [1 << 17]byte + var x6443 [1 << 17]byte + var x6444 [1 << 17]byte + var x6445 [1 << 17]byte + var x6446 [1 << 17]byte + var x6447 [1 << 17]byte + var x6448 [1 << 17]byte + var x6449 [1 << 17]byte + var x6450 [1 << 17]byte + var x6451 [1 << 17]byte + var x6452 [1 << 17]byte + var x6453 [1 << 17]byte + var x6454 [1 << 17]byte + var x6455 [1 << 17]byte + var x6456 [1 << 17]byte + var x6457 [1 << 17]byte + var x6458 [1 << 17]byte + var x6459 [1 << 17]byte + var x6460 [1 << 17]byte + var x6461 [1 << 17]byte + var x6462 [1 << 17]byte + var x6463 [1 << 17]byte + var x6464 [1 << 17]byte + var x6465 [1 << 17]byte + var x6466 [1 << 17]byte + var x6467 [1 << 17]byte + var x6468 [1 << 17]byte + var x6469 [1 << 17]byte + var x6470 [1 << 17]byte + var x6471 [1 << 17]byte + var x6472 [1 << 17]byte + var x6473 [1 << 17]byte + var x6474 [1 << 17]byte + var x6475 [1 << 17]byte + var x6476 [1 << 17]byte + var x6477 [1 << 17]byte + var x6478 [1 << 17]byte + var x6479 [1 << 17]byte + var x6480 [1 << 17]byte + var x6481 [1 << 17]byte + var x6482 [1 << 17]byte + var x6483 [1 << 17]byte + var x6484 [1 << 17]byte + var x6485 [1 << 17]byte + var x6486 [1 << 17]byte + var x6487 [1 << 17]byte + var x6488 [1 << 17]byte + var x6489 [1 << 17]byte + var x6490 [1 << 17]byte + var x6491 [1 << 17]byte + var x6492 [1 << 17]byte + var x6493 [1 << 17]byte + var x6494 [1 << 17]byte + var x6495 [1 << 17]byte + var x6496 [1 << 17]byte + var x6497 [1 << 17]byte + var x6498 [1 << 17]byte + var x6499 [1 << 17]byte + var x6500 [1 << 17]byte + var x6501 [1 << 17]byte + var x6502 [1 << 17]byte + var x6503 [1 << 17]byte + var x6504 [1 << 17]byte + var x6505 [1 << 17]byte + var x6506 [1 << 17]byte + var x6507 [1 << 17]byte + var x6508 [1 << 17]byte + var x6509 [1 << 17]byte + var x6510 [1 << 17]byte + var x6511 [1 << 17]byte + var x6512 [1 << 17]byte + var x6513 [1 << 17]byte + var x6514 [1 << 17]byte + var x6515 [1 << 17]byte + var x6516 [1 << 17]byte + var x6517 [1 << 17]byte + var x6518 [1 << 17]byte + var x6519 [1 << 17]byte + var x6520 [1 << 17]byte + var x6521 [1 << 17]byte + var x6522 [1 << 17]byte + var x6523 [1 << 17]byte + var x6524 [1 << 17]byte + var x6525 [1 << 17]byte + var x6526 [1 << 17]byte + var x6527 [1 << 17]byte + var x6528 [1 << 17]byte + var x6529 [1 << 17]byte + var x6530 [1 << 17]byte + var x6531 [1 << 17]byte + var x6532 [1 << 17]byte + var x6533 [1 << 17]byte + var x6534 [1 << 17]byte + var x6535 [1 << 17]byte + var x6536 [1 << 17]byte + var x6537 [1 << 17]byte + var x6538 [1 << 17]byte + var x6539 [1 << 17]byte + var x6540 [1 << 17]byte + var x6541 [1 << 17]byte + var x6542 [1 << 17]byte + var x6543 [1 << 17]byte + var x6544 [1 << 17]byte + var x6545 [1 << 17]byte + var x6546 [1 << 17]byte + var x6547 [1 << 17]byte + var x6548 [1 << 17]byte + var x6549 [1 << 17]byte + var x6550 [1 << 17]byte + var x6551 [1 << 17]byte + var x6552 [1 << 17]byte + var x6553 [1 << 17]byte + var x6554 [1 << 17]byte + var x6555 [1 << 17]byte + var x6556 [1 << 17]byte + var x6557 [1 << 17]byte + var x6558 [1 << 17]byte + var x6559 [1 << 17]byte + var x6560 [1 << 17]byte + var x6561 [1 << 17]byte + var x6562 [1 << 17]byte + var x6563 [1 << 17]byte + var x6564 [1 << 17]byte + var x6565 [1 << 17]byte + var x6566 [1 << 17]byte + var x6567 [1 << 17]byte + var x6568 [1 << 17]byte + var x6569 [1 << 17]byte + var x6570 [1 << 17]byte + var x6571 [1 << 17]byte + var x6572 [1 << 17]byte + var x6573 [1 << 17]byte + var x6574 [1 << 17]byte + var x6575 [1 << 17]byte + var x6576 [1 << 17]byte + var x6577 [1 << 17]byte + var x6578 [1 << 17]byte + var x6579 [1 << 17]byte + var x6580 [1 << 17]byte + var x6581 [1 << 17]byte + var x6582 [1 << 17]byte + var x6583 [1 << 17]byte + var x6584 [1 << 17]byte + var x6585 [1 << 17]byte + var x6586 [1 << 17]byte + var x6587 [1 << 17]byte + var x6588 [1 << 17]byte + var x6589 [1 << 17]byte + var x6590 [1 << 17]byte + var x6591 [1 << 17]byte + var x6592 [1 << 17]byte + var x6593 [1 << 17]byte + var x6594 [1 << 17]byte + var x6595 [1 << 17]byte + var x6596 [1 << 17]byte + var x6597 [1 << 17]byte + var x6598 [1 << 17]byte + var x6599 [1 << 17]byte + var x6600 [1 << 17]byte + var x6601 [1 << 17]byte + var x6602 [1 << 17]byte + var x6603 [1 << 17]byte + var x6604 [1 << 17]byte + var x6605 [1 << 17]byte + var x6606 [1 << 17]byte + var x6607 [1 << 17]byte + var x6608 [1 << 17]byte + var x6609 [1 << 17]byte + var x6610 [1 << 17]byte + var x6611 [1 << 17]byte + var x6612 [1 << 17]byte + var x6613 [1 << 17]byte + var x6614 [1 << 17]byte + var x6615 [1 << 17]byte + var x6616 [1 << 17]byte + var x6617 [1 << 17]byte + var x6618 [1 << 17]byte + var x6619 [1 << 17]byte + var x6620 [1 << 17]byte + var x6621 [1 << 17]byte + var x6622 [1 << 17]byte + var x6623 [1 << 17]byte + var x6624 [1 << 17]byte + var x6625 [1 << 17]byte + var x6626 [1 << 17]byte + var x6627 [1 << 17]byte + var x6628 [1 << 17]byte + var x6629 [1 << 17]byte + var x6630 [1 << 17]byte + var x6631 [1 << 17]byte + var x6632 [1 << 17]byte + var x6633 [1 << 17]byte + var x6634 [1 << 17]byte + var x6635 [1 << 17]byte + var x6636 [1 << 17]byte + var x6637 [1 << 17]byte + var x6638 [1 << 17]byte + var x6639 [1 << 17]byte + var x6640 [1 << 17]byte + var x6641 [1 << 17]byte + var x6642 [1 << 17]byte + var x6643 [1 << 17]byte + var x6644 [1 << 17]byte + var x6645 [1 << 17]byte + var x6646 [1 << 17]byte + var x6647 [1 << 17]byte + var x6648 [1 << 17]byte + var x6649 [1 << 17]byte + var x6650 [1 << 17]byte + var x6651 [1 << 17]byte + var x6652 [1 << 17]byte + var x6653 [1 << 17]byte + var x6654 [1 << 17]byte + var x6655 [1 << 17]byte + var x6656 [1 << 17]byte + var x6657 [1 << 17]byte + var x6658 [1 << 17]byte + var x6659 [1 << 17]byte + var x6660 [1 << 17]byte + var x6661 [1 << 17]byte + var x6662 [1 << 17]byte + var x6663 [1 << 17]byte + var x6664 [1 << 17]byte + var x6665 [1 << 17]byte + var x6666 [1 << 17]byte + var x6667 [1 << 17]byte + var x6668 [1 << 17]byte + var x6669 [1 << 17]byte + var x6670 [1 << 17]byte + var x6671 [1 << 17]byte + var x6672 [1 << 17]byte + var x6673 [1 << 17]byte + var x6674 [1 << 17]byte + var x6675 [1 << 17]byte + var x6676 [1 << 17]byte + var x6677 [1 << 17]byte + var x6678 [1 << 17]byte + var x6679 [1 << 17]byte + var x6680 [1 << 17]byte + var x6681 [1 << 17]byte + var x6682 [1 << 17]byte + var x6683 [1 << 17]byte + var x6684 [1 << 17]byte + var x6685 [1 << 17]byte + var x6686 [1 << 17]byte + var x6687 [1 << 17]byte + var x6688 [1 << 17]byte + var x6689 [1 << 17]byte + var x6690 [1 << 17]byte + var x6691 [1 << 17]byte + var x6692 [1 << 17]byte + var x6693 [1 << 17]byte + var x6694 [1 << 17]byte + var x6695 [1 << 17]byte + var x6696 [1 << 17]byte + var x6697 [1 << 17]byte + var x6698 [1 << 17]byte + var x6699 [1 << 17]byte + var x6700 [1 << 17]byte + var x6701 [1 << 17]byte + var x6702 [1 << 17]byte + var x6703 [1 << 17]byte + var x6704 [1 << 17]byte + var x6705 [1 << 17]byte + var x6706 [1 << 17]byte + var x6707 [1 << 17]byte + var x6708 [1 << 17]byte + var x6709 [1 << 17]byte + var x6710 [1 << 17]byte + var x6711 [1 << 17]byte + var x6712 [1 << 17]byte + var x6713 [1 << 17]byte + var x6714 [1 << 17]byte + var x6715 [1 << 17]byte + var x6716 [1 << 17]byte + var x6717 [1 << 17]byte + var x6718 [1 << 17]byte + var x6719 [1 << 17]byte + var x6720 [1 << 17]byte + var x6721 [1 << 17]byte + var x6722 [1 << 17]byte + var x6723 [1 << 17]byte + var x6724 [1 << 17]byte + var x6725 [1 << 17]byte + var x6726 [1 << 17]byte + var x6727 [1 << 17]byte + var x6728 [1 << 17]byte + var x6729 [1 << 17]byte + var x6730 [1 << 17]byte + var x6731 [1 << 17]byte + var x6732 [1 << 17]byte + var x6733 [1 << 17]byte + var x6734 [1 << 17]byte + var x6735 [1 << 17]byte + var x6736 [1 << 17]byte + var x6737 [1 << 17]byte + var x6738 [1 << 17]byte + var x6739 [1 << 17]byte + var x6740 [1 << 17]byte + var x6741 [1 << 17]byte + var x6742 [1 << 17]byte + var x6743 [1 << 17]byte + var x6744 [1 << 17]byte + var x6745 [1 << 17]byte + var x6746 [1 << 17]byte + var x6747 [1 << 17]byte + var x6748 [1 << 17]byte + var x6749 [1 << 17]byte + var x6750 [1 << 17]byte + var x6751 [1 << 17]byte + var x6752 [1 << 17]byte + var x6753 [1 << 17]byte + var x6754 [1 << 17]byte + var x6755 [1 << 17]byte + var x6756 [1 << 17]byte + var x6757 [1 << 17]byte + var x6758 [1 << 17]byte + var x6759 [1 << 17]byte + var x6760 [1 << 17]byte + var x6761 [1 << 17]byte + var x6762 [1 << 17]byte + var x6763 [1 << 17]byte + var x6764 [1 << 17]byte + var x6765 [1 << 17]byte + var x6766 [1 << 17]byte + var x6767 [1 << 17]byte + var x6768 [1 << 17]byte + var x6769 [1 << 17]byte + var x6770 [1 << 17]byte + var x6771 [1 << 17]byte + var x6772 [1 << 17]byte + var x6773 [1 << 17]byte + var x6774 [1 << 17]byte + var x6775 [1 << 17]byte + var x6776 [1 << 17]byte + var x6777 [1 << 17]byte + var x6778 [1 << 17]byte + var x6779 [1 << 17]byte + var x6780 [1 << 17]byte + var x6781 [1 << 17]byte + var x6782 [1 << 17]byte + var x6783 [1 << 17]byte + var x6784 [1 << 17]byte + var x6785 [1 << 17]byte + var x6786 [1 << 17]byte + var x6787 [1 << 17]byte + var x6788 [1 << 17]byte + var x6789 [1 << 17]byte + var x6790 [1 << 17]byte + var x6791 [1 << 17]byte + var x6792 [1 << 17]byte + var x6793 [1 << 17]byte + var x6794 [1 << 17]byte + var x6795 [1 << 17]byte + var x6796 [1 << 17]byte + var x6797 [1 << 17]byte + var x6798 [1 << 17]byte + var x6799 [1 << 17]byte + var x6800 [1 << 17]byte + var x6801 [1 << 17]byte + var x6802 [1 << 17]byte + var x6803 [1 << 17]byte + var x6804 [1 << 17]byte + var x6805 [1 << 17]byte + var x6806 [1 << 17]byte + var x6807 [1 << 17]byte + var x6808 [1 << 17]byte + var x6809 [1 << 17]byte + var x6810 [1 << 17]byte + var x6811 [1 << 17]byte + var x6812 [1 << 17]byte + var x6813 [1 << 17]byte + var x6814 [1 << 17]byte + var x6815 [1 << 17]byte + var x6816 [1 << 17]byte + var x6817 [1 << 17]byte + var x6818 [1 << 17]byte + var x6819 [1 << 17]byte + var x6820 [1 << 17]byte + var x6821 [1 << 17]byte + var x6822 [1 << 17]byte + var x6823 [1 << 17]byte + var x6824 [1 << 17]byte + var x6825 [1 << 17]byte + var x6826 [1 << 17]byte + var x6827 [1 << 17]byte + var x6828 [1 << 17]byte + var x6829 [1 << 17]byte + var x6830 [1 << 17]byte + var x6831 [1 << 17]byte + var x6832 [1 << 17]byte + var x6833 [1 << 17]byte + var x6834 [1 << 17]byte + var x6835 [1 << 17]byte + var x6836 [1 << 17]byte + var x6837 [1 << 17]byte + var x6838 [1 << 17]byte + var x6839 [1 << 17]byte + var x6840 [1 << 17]byte + var x6841 [1 << 17]byte + var x6842 [1 << 17]byte + var x6843 [1 << 17]byte + var x6844 [1 << 17]byte + var x6845 [1 << 17]byte + var x6846 [1 << 17]byte + var x6847 [1 << 17]byte + var x6848 [1 << 17]byte + var x6849 [1 << 17]byte + var x6850 [1 << 17]byte + var x6851 [1 << 17]byte + var x6852 [1 << 17]byte + var x6853 [1 << 17]byte + var x6854 [1 << 17]byte + var x6855 [1 << 17]byte + var x6856 [1 << 17]byte + var x6857 [1 << 17]byte + var x6858 [1 << 17]byte + var x6859 [1 << 17]byte + var x6860 [1 << 17]byte + var x6861 [1 << 17]byte + var x6862 [1 << 17]byte + var x6863 [1 << 17]byte + var x6864 [1 << 17]byte + var x6865 [1 << 17]byte + var x6866 [1 << 17]byte + var x6867 [1 << 17]byte + var x6868 [1 << 17]byte + var x6869 [1 << 17]byte + var x6870 [1 << 17]byte + var x6871 [1 << 17]byte + var x6872 [1 << 17]byte + var x6873 [1 << 17]byte + var x6874 [1 << 17]byte + var x6875 [1 << 17]byte + var x6876 [1 << 17]byte + var x6877 [1 << 17]byte + var x6878 [1 << 17]byte + var x6879 [1 << 17]byte + var x6880 [1 << 17]byte + var x6881 [1 << 17]byte + var x6882 [1 << 17]byte + var x6883 [1 << 17]byte + var x6884 [1 << 17]byte + var x6885 [1 << 17]byte + var x6886 [1 << 17]byte + var x6887 [1 << 17]byte + var x6888 [1 << 17]byte + var x6889 [1 << 17]byte + var x6890 [1 << 17]byte + var x6891 [1 << 17]byte + var x6892 [1 << 17]byte + var x6893 [1 << 17]byte + var x6894 [1 << 17]byte + var x6895 [1 << 17]byte + var x6896 [1 << 17]byte + var x6897 [1 << 17]byte + var x6898 [1 << 17]byte + var x6899 [1 << 17]byte + var x6900 [1 << 17]byte + var x6901 [1 << 17]byte + var x6902 [1 << 17]byte + var x6903 [1 << 17]byte + var x6904 [1 << 17]byte + var x6905 [1 << 17]byte + var x6906 [1 << 17]byte + var x6907 [1 << 17]byte + var x6908 [1 << 17]byte + var x6909 [1 << 17]byte + var x6910 [1 << 17]byte + var x6911 [1 << 17]byte + var x6912 [1 << 17]byte + var x6913 [1 << 17]byte + var x6914 [1 << 17]byte + var x6915 [1 << 17]byte + var x6916 [1 << 17]byte + var x6917 [1 << 17]byte + var x6918 [1 << 17]byte + var x6919 [1 << 17]byte + var x6920 [1 << 17]byte + var x6921 [1 << 17]byte + var x6922 [1 << 17]byte + var x6923 [1 << 17]byte + var x6924 [1 << 17]byte + var x6925 [1 << 17]byte + var x6926 [1 << 17]byte + var x6927 [1 << 17]byte + var x6928 [1 << 17]byte + var x6929 [1 << 17]byte + var x6930 [1 << 17]byte + var x6931 [1 << 17]byte + var x6932 [1 << 17]byte + var x6933 [1 << 17]byte + var x6934 [1 << 17]byte + var x6935 [1 << 17]byte + var x6936 [1 << 17]byte + var x6937 [1 << 17]byte + var x6938 [1 << 17]byte + var x6939 [1 << 17]byte + var x6940 [1 << 17]byte + var x6941 [1 << 17]byte + var x6942 [1 << 17]byte + var x6943 [1 << 17]byte + var x6944 [1 << 17]byte + var x6945 [1 << 17]byte + var x6946 [1 << 17]byte + var x6947 [1 << 17]byte + var x6948 [1 << 17]byte + var x6949 [1 << 17]byte + var x6950 [1 << 17]byte + var x6951 [1 << 17]byte + var x6952 [1 << 17]byte + var x6953 [1 << 17]byte + var x6954 [1 << 17]byte + var x6955 [1 << 17]byte + var x6956 [1 << 17]byte + var x6957 [1 << 17]byte + var x6958 [1 << 17]byte + var x6959 [1 << 17]byte + var x6960 [1 << 17]byte + var x6961 [1 << 17]byte + var x6962 [1 << 17]byte + var x6963 [1 << 17]byte + var x6964 [1 << 17]byte + var x6965 [1 << 17]byte + var x6966 [1 << 17]byte + var x6967 [1 << 17]byte + var x6968 [1 << 17]byte + var x6969 [1 << 17]byte + var x6970 [1 << 17]byte + var x6971 [1 << 17]byte + var x6972 [1 << 17]byte + var x6973 [1 << 17]byte + var x6974 [1 << 17]byte + var x6975 [1 << 17]byte + var x6976 [1 << 17]byte + var x6977 [1 << 17]byte + var x6978 [1 << 17]byte + var x6979 [1 << 17]byte + var x6980 [1 << 17]byte + var x6981 [1 << 17]byte + var x6982 [1 << 17]byte + var x6983 [1 << 17]byte + var x6984 [1 << 17]byte + var x6985 [1 << 17]byte + var x6986 [1 << 17]byte + var x6987 [1 << 17]byte + var x6988 [1 << 17]byte + var x6989 [1 << 17]byte + var x6990 [1 << 17]byte + var x6991 [1 << 17]byte + var x6992 [1 << 17]byte + var x6993 [1 << 17]byte + var x6994 [1 << 17]byte + var x6995 [1 << 17]byte + var x6996 [1 << 17]byte + var x6997 [1 << 17]byte + var x6998 [1 << 17]byte + var x6999 [1 << 17]byte + var x7000 [1 << 17]byte + var x7001 [1 << 17]byte + var x7002 [1 << 17]byte + var x7003 [1 << 17]byte + var x7004 [1 << 17]byte + var x7005 [1 << 17]byte + var x7006 [1 << 17]byte + var x7007 [1 << 17]byte + var x7008 [1 << 17]byte + var x7009 [1 << 17]byte + var x7010 [1 << 17]byte + var x7011 [1 << 17]byte + var x7012 [1 << 17]byte + var x7013 [1 << 17]byte + var x7014 [1 << 17]byte + var x7015 [1 << 17]byte + var x7016 [1 << 17]byte + var x7017 [1 << 17]byte + var x7018 [1 << 17]byte + var x7019 [1 << 17]byte + var x7020 [1 << 17]byte + var x7021 [1 << 17]byte + var x7022 [1 << 17]byte + var x7023 [1 << 17]byte + var x7024 [1 << 17]byte + var x7025 [1 << 17]byte + var x7026 [1 << 17]byte + var x7027 [1 << 17]byte + var x7028 [1 << 17]byte + var x7029 [1 << 17]byte + var x7030 [1 << 17]byte + var x7031 [1 << 17]byte + var x7032 [1 << 17]byte + var x7033 [1 << 17]byte + var x7034 [1 << 17]byte + var x7035 [1 << 17]byte + var x7036 [1 << 17]byte + var x7037 [1 << 17]byte + var x7038 [1 << 17]byte + var x7039 [1 << 17]byte + var x7040 [1 << 17]byte + var x7041 [1 << 17]byte + var x7042 [1 << 17]byte + var x7043 [1 << 17]byte + var x7044 [1 << 17]byte + var x7045 [1 << 17]byte + var x7046 [1 << 17]byte + var x7047 [1 << 17]byte + var x7048 [1 << 17]byte + var x7049 [1 << 17]byte + var x7050 [1 << 17]byte + var x7051 [1 << 17]byte + var x7052 [1 << 17]byte + var x7053 [1 << 17]byte + var x7054 [1 << 17]byte + var x7055 [1 << 17]byte + var x7056 [1 << 17]byte + var x7057 [1 << 17]byte + var x7058 [1 << 17]byte + var x7059 [1 << 17]byte + var x7060 [1 << 17]byte + var x7061 [1 << 17]byte + var x7062 [1 << 17]byte + var x7063 [1 << 17]byte + var x7064 [1 << 17]byte + var x7065 [1 << 17]byte + var x7066 [1 << 17]byte + var x7067 [1 << 17]byte + var x7068 [1 << 17]byte + var x7069 [1 << 17]byte + var x7070 [1 << 17]byte + var x7071 [1 << 17]byte + var x7072 [1 << 17]byte + var x7073 [1 << 17]byte + var x7074 [1 << 17]byte + var x7075 [1 << 17]byte + var x7076 [1 << 17]byte + var x7077 [1 << 17]byte + var x7078 [1 << 17]byte + var x7079 [1 << 17]byte + var x7080 [1 << 17]byte + var x7081 [1 << 17]byte + var x7082 [1 << 17]byte + var x7083 [1 << 17]byte + var x7084 [1 << 17]byte + var x7085 [1 << 17]byte + var x7086 [1 << 17]byte + var x7087 [1 << 17]byte + var x7088 [1 << 17]byte + var x7089 [1 << 17]byte + var x7090 [1 << 17]byte + var x7091 [1 << 17]byte + var x7092 [1 << 17]byte + var x7093 [1 << 17]byte + var x7094 [1 << 17]byte + var x7095 [1 << 17]byte + var x7096 [1 << 17]byte + var x7097 [1 << 17]byte + var x7098 [1 << 17]byte + var x7099 [1 << 17]byte + var x7100 [1 << 17]byte + var x7101 [1 << 17]byte + var x7102 [1 << 17]byte + var x7103 [1 << 17]byte + var x7104 [1 << 17]byte + var x7105 [1 << 17]byte + var x7106 [1 << 17]byte + var x7107 [1 << 17]byte + var x7108 [1 << 17]byte + var x7109 [1 << 17]byte + var x7110 [1 << 17]byte + var x7111 [1 << 17]byte + var x7112 [1 << 17]byte + var x7113 [1 << 17]byte + var x7114 [1 << 17]byte + var x7115 [1 << 17]byte + var x7116 [1 << 17]byte + var x7117 [1 << 17]byte + var x7118 [1 << 17]byte + var x7119 [1 << 17]byte + var x7120 [1 << 17]byte + var x7121 [1 << 17]byte + var x7122 [1 << 17]byte + var x7123 [1 << 17]byte + var x7124 [1 << 17]byte + var x7125 [1 << 17]byte + var x7126 [1 << 17]byte + var x7127 [1 << 17]byte + var x7128 [1 << 17]byte + var x7129 [1 << 17]byte + var x7130 [1 << 17]byte + var x7131 [1 << 17]byte + var x7132 [1 << 17]byte + var x7133 [1 << 17]byte + var x7134 [1 << 17]byte + var x7135 [1 << 17]byte + var x7136 [1 << 17]byte + var x7137 [1 << 17]byte + var x7138 [1 << 17]byte + var x7139 [1 << 17]byte + var x7140 [1 << 17]byte + var x7141 [1 << 17]byte + var x7142 [1 << 17]byte + var x7143 [1 << 17]byte + var x7144 [1 << 17]byte + var x7145 [1 << 17]byte + var x7146 [1 << 17]byte + var x7147 [1 << 17]byte + var x7148 [1 << 17]byte + var x7149 [1 << 17]byte + var x7150 [1 << 17]byte + var x7151 [1 << 17]byte + var x7152 [1 << 17]byte + var x7153 [1 << 17]byte + var x7154 [1 << 17]byte + var x7155 [1 << 17]byte + var x7156 [1 << 17]byte + var x7157 [1 << 17]byte + var x7158 [1 << 17]byte + var x7159 [1 << 17]byte + var x7160 [1 << 17]byte + var x7161 [1 << 17]byte + var x7162 [1 << 17]byte + var x7163 [1 << 17]byte + var x7164 [1 << 17]byte + var x7165 [1 << 17]byte + var x7166 [1 << 17]byte + var x7167 [1 << 17]byte + var x7168 [1 << 17]byte + var x7169 [1 << 17]byte + var x7170 [1 << 17]byte + var x7171 [1 << 17]byte + var x7172 [1 << 17]byte + var x7173 [1 << 17]byte + var x7174 [1 << 17]byte + var x7175 [1 << 17]byte + var x7176 [1 << 17]byte + var x7177 [1 << 17]byte + var x7178 [1 << 17]byte + var x7179 [1 << 17]byte + var x7180 [1 << 17]byte + var x7181 [1 << 17]byte + var x7182 [1 << 17]byte + var x7183 [1 << 17]byte + var x7184 [1 << 17]byte + var x7185 [1 << 17]byte + var x7186 [1 << 17]byte + var x7187 [1 << 17]byte + var x7188 [1 << 17]byte + var x7189 [1 << 17]byte + var x7190 [1 << 17]byte + var x7191 [1 << 17]byte + var x7192 [1 << 17]byte + var x7193 [1 << 17]byte + var x7194 [1 << 17]byte + var x7195 [1 << 17]byte + var x7196 [1 << 17]byte + var x7197 [1 << 17]byte + var x7198 [1 << 17]byte + var x7199 [1 << 17]byte + var x7200 [1 << 17]byte + var x7201 [1 << 17]byte + var x7202 [1 << 17]byte + var x7203 [1 << 17]byte + var x7204 [1 << 17]byte + var x7205 [1 << 17]byte + var x7206 [1 << 17]byte + var x7207 [1 << 17]byte + var x7208 [1 << 17]byte + var x7209 [1 << 17]byte + var x7210 [1 << 17]byte + var x7211 [1 << 17]byte + var x7212 [1 << 17]byte + var x7213 [1 << 17]byte + var x7214 [1 << 17]byte + var x7215 [1 << 17]byte + var x7216 [1 << 17]byte + var x7217 [1 << 17]byte + var x7218 [1 << 17]byte + var x7219 [1 << 17]byte + var x7220 [1 << 17]byte + var x7221 [1 << 17]byte + var x7222 [1 << 17]byte + var x7223 [1 << 17]byte + var x7224 [1 << 17]byte + var x7225 [1 << 17]byte + var x7226 [1 << 17]byte + var x7227 [1 << 17]byte + var x7228 [1 << 17]byte + var x7229 [1 << 17]byte + var x7230 [1 << 17]byte + var x7231 [1 << 17]byte + var x7232 [1 << 17]byte + var x7233 [1 << 17]byte + var x7234 [1 << 17]byte + var x7235 [1 << 17]byte + var x7236 [1 << 17]byte + var x7237 [1 << 17]byte + var x7238 [1 << 17]byte + var x7239 [1 << 17]byte + var x7240 [1 << 17]byte + var x7241 [1 << 17]byte + var x7242 [1 << 17]byte + var x7243 [1 << 17]byte + var x7244 [1 << 17]byte + var x7245 [1 << 17]byte + var x7246 [1 << 17]byte + var x7247 [1 << 17]byte + var x7248 [1 << 17]byte + var x7249 [1 << 17]byte + var x7250 [1 << 17]byte + var x7251 [1 << 17]byte + var x7252 [1 << 17]byte + var x7253 [1 << 17]byte + var x7254 [1 << 17]byte + var x7255 [1 << 17]byte + var x7256 [1 << 17]byte + var x7257 [1 << 17]byte + var x7258 [1 << 17]byte + var x7259 [1 << 17]byte + var x7260 [1 << 17]byte + var x7261 [1 << 17]byte + var x7262 [1 << 17]byte + var x7263 [1 << 17]byte + var x7264 [1 << 17]byte + var x7265 [1 << 17]byte + var x7266 [1 << 17]byte + var x7267 [1 << 17]byte + var x7268 [1 << 17]byte + var x7269 [1 << 17]byte + var x7270 [1 << 17]byte + var x7271 [1 << 17]byte + var x7272 [1 << 17]byte + var x7273 [1 << 17]byte + var x7274 [1 << 17]byte + var x7275 [1 << 17]byte + var x7276 [1 << 17]byte + var x7277 [1 << 17]byte + var x7278 [1 << 17]byte + var x7279 [1 << 17]byte + var x7280 [1 << 17]byte + var x7281 [1 << 17]byte + var x7282 [1 << 17]byte + var x7283 [1 << 17]byte + var x7284 [1 << 17]byte + var x7285 [1 << 17]byte + var x7286 [1 << 17]byte + var x7287 [1 << 17]byte + var x7288 [1 << 17]byte + var x7289 [1 << 17]byte + var x7290 [1 << 17]byte + var x7291 [1 << 17]byte + var x7292 [1 << 17]byte + var x7293 [1 << 17]byte + var x7294 [1 << 17]byte + var x7295 [1 << 17]byte + var x7296 [1 << 17]byte + var x7297 [1 << 17]byte + var x7298 [1 << 17]byte + var x7299 [1 << 17]byte + var x7300 [1 << 17]byte + var x7301 [1 << 17]byte + var x7302 [1 << 17]byte + var x7303 [1 << 17]byte + var x7304 [1 << 17]byte + var x7305 [1 << 17]byte + var x7306 [1 << 17]byte + var x7307 [1 << 17]byte + var x7308 [1 << 17]byte + var x7309 [1 << 17]byte + var x7310 [1 << 17]byte + var x7311 [1 << 17]byte + var x7312 [1 << 17]byte + var x7313 [1 << 17]byte + var x7314 [1 << 17]byte + var x7315 [1 << 17]byte + var x7316 [1 << 17]byte + var x7317 [1 << 17]byte + var x7318 [1 << 17]byte + var x7319 [1 << 17]byte + var x7320 [1 << 17]byte + var x7321 [1 << 17]byte + var x7322 [1 << 17]byte + var x7323 [1 << 17]byte + var x7324 [1 << 17]byte + var x7325 [1 << 17]byte + var x7326 [1 << 17]byte + var x7327 [1 << 17]byte + var x7328 [1 << 17]byte + var x7329 [1 << 17]byte + var x7330 [1 << 17]byte + var x7331 [1 << 17]byte + var x7332 [1 << 17]byte + var x7333 [1 << 17]byte + var x7334 [1 << 17]byte + var x7335 [1 << 17]byte + var x7336 [1 << 17]byte + var x7337 [1 << 17]byte + var x7338 [1 << 17]byte + var x7339 [1 << 17]byte + var x7340 [1 << 17]byte + var x7341 [1 << 17]byte + var x7342 [1 << 17]byte + var x7343 [1 << 17]byte + var x7344 [1 << 17]byte + var x7345 [1 << 17]byte + var x7346 [1 << 17]byte + var x7347 [1 << 17]byte + var x7348 [1 << 17]byte + var x7349 [1 << 17]byte + var x7350 [1 << 17]byte + var x7351 [1 << 17]byte + var x7352 [1 << 17]byte + var x7353 [1 << 17]byte + var x7354 [1 << 17]byte + var x7355 [1 << 17]byte + var x7356 [1 << 17]byte + var x7357 [1 << 17]byte + var x7358 [1 << 17]byte + var x7359 [1 << 17]byte + var x7360 [1 << 17]byte + var x7361 [1 << 17]byte + var x7362 [1 << 17]byte + var x7363 [1 << 17]byte + var x7364 [1 << 17]byte + var x7365 [1 << 17]byte + var x7366 [1 << 17]byte + var x7367 [1 << 17]byte + var x7368 [1 << 17]byte + var x7369 [1 << 17]byte + var x7370 [1 << 17]byte + var x7371 [1 << 17]byte + var x7372 [1 << 17]byte + var x7373 [1 << 17]byte + var x7374 [1 << 17]byte + var x7375 [1 << 17]byte + var x7376 [1 << 17]byte + var x7377 [1 << 17]byte + var x7378 [1 << 17]byte + var x7379 [1 << 17]byte + var x7380 [1 << 17]byte + var x7381 [1 << 17]byte + var x7382 [1 << 17]byte + var x7383 [1 << 17]byte + var x7384 [1 << 17]byte + var x7385 [1 << 17]byte + var x7386 [1 << 17]byte + var x7387 [1 << 17]byte + var x7388 [1 << 17]byte + var x7389 [1 << 17]byte + var x7390 [1 << 17]byte + var x7391 [1 << 17]byte + var x7392 [1 << 17]byte + var x7393 [1 << 17]byte + var x7394 [1 << 17]byte + var x7395 [1 << 17]byte + var x7396 [1 << 17]byte + var x7397 [1 << 17]byte + var x7398 [1 << 17]byte + var x7399 [1 << 17]byte + var x7400 [1 << 17]byte + var x7401 [1 << 17]byte + var x7402 [1 << 17]byte + var x7403 [1 << 17]byte + var x7404 [1 << 17]byte + var x7405 [1 << 17]byte + var x7406 [1 << 17]byte + var x7407 [1 << 17]byte + var x7408 [1 << 17]byte + var x7409 [1 << 17]byte + var x7410 [1 << 17]byte + var x7411 [1 << 17]byte + var x7412 [1 << 17]byte + var x7413 [1 << 17]byte + var x7414 [1 << 17]byte + var x7415 [1 << 17]byte + var x7416 [1 << 17]byte + var x7417 [1 << 17]byte + var x7418 [1 << 17]byte + var x7419 [1 << 17]byte + var x7420 [1 << 17]byte + var x7421 [1 << 17]byte + var x7422 [1 << 17]byte + var x7423 [1 << 17]byte + var x7424 [1 << 17]byte + var x7425 [1 << 17]byte + var x7426 [1 << 17]byte + var x7427 [1 << 17]byte + var x7428 [1 << 17]byte + var x7429 [1 << 17]byte + var x7430 [1 << 17]byte + var x7431 [1 << 17]byte + var x7432 [1 << 17]byte + var x7433 [1 << 17]byte + var x7434 [1 << 17]byte + var x7435 [1 << 17]byte + var x7436 [1 << 17]byte + var x7437 [1 << 17]byte + var x7438 [1 << 17]byte + var x7439 [1 << 17]byte + var x7440 [1 << 17]byte + var x7441 [1 << 17]byte + var x7442 [1 << 17]byte + var x7443 [1 << 17]byte + var x7444 [1 << 17]byte + var x7445 [1 << 17]byte + var x7446 [1 << 17]byte + var x7447 [1 << 17]byte + var x7448 [1 << 17]byte + var x7449 [1 << 17]byte + var x7450 [1 << 17]byte + var x7451 [1 << 17]byte + var x7452 [1 << 17]byte + var x7453 [1 << 17]byte + var x7454 [1 << 17]byte + var x7455 [1 << 17]byte + var x7456 [1 << 17]byte + var x7457 [1 << 17]byte + var x7458 [1 << 17]byte + var x7459 [1 << 17]byte + var x7460 [1 << 17]byte + var x7461 [1 << 17]byte + var x7462 [1 << 17]byte + var x7463 [1 << 17]byte + var x7464 [1 << 17]byte + var x7465 [1 << 17]byte + var x7466 [1 << 17]byte + var x7467 [1 << 17]byte + var x7468 [1 << 17]byte + var x7469 [1 << 17]byte + var x7470 [1 << 17]byte + var x7471 [1 << 17]byte + var x7472 [1 << 17]byte + var x7473 [1 << 17]byte + var x7474 [1 << 17]byte + var x7475 [1 << 17]byte + var x7476 [1 << 17]byte + var x7477 [1 << 17]byte + var x7478 [1 << 17]byte + var x7479 [1 << 17]byte + var x7480 [1 << 17]byte + var x7481 [1 << 17]byte + var x7482 [1 << 17]byte + var x7483 [1 << 17]byte + var x7484 [1 << 17]byte + var x7485 [1 << 17]byte + var x7486 [1 << 17]byte + var x7487 [1 << 17]byte + var x7488 [1 << 17]byte + var x7489 [1 << 17]byte + var x7490 [1 << 17]byte + var x7491 [1 << 17]byte + var x7492 [1 << 17]byte + var x7493 [1 << 17]byte + var x7494 [1 << 17]byte + var x7495 [1 << 17]byte + var x7496 [1 << 17]byte + var x7497 [1 << 17]byte + var x7498 [1 << 17]byte + var x7499 [1 << 17]byte + var x7500 [1 << 17]byte + var x7501 [1 << 17]byte + var x7502 [1 << 17]byte + var x7503 [1 << 17]byte + var x7504 [1 << 17]byte + var x7505 [1 << 17]byte + var x7506 [1 << 17]byte + var x7507 [1 << 17]byte + var x7508 [1 << 17]byte + var x7509 [1 << 17]byte + var x7510 [1 << 17]byte + var x7511 [1 << 17]byte + var x7512 [1 << 17]byte + var x7513 [1 << 17]byte + var x7514 [1 << 17]byte + var x7515 [1 << 17]byte + var x7516 [1 << 17]byte + var x7517 [1 << 17]byte + var x7518 [1 << 17]byte + var x7519 [1 << 17]byte + var x7520 [1 << 17]byte + var x7521 [1 << 17]byte + var x7522 [1 << 17]byte + var x7523 [1 << 17]byte + var x7524 [1 << 17]byte + var x7525 [1 << 17]byte + var x7526 [1 << 17]byte + var x7527 [1 << 17]byte + var x7528 [1 << 17]byte + var x7529 [1 << 17]byte + var x7530 [1 << 17]byte + var x7531 [1 << 17]byte + var x7532 [1 << 17]byte + var x7533 [1 << 17]byte + var x7534 [1 << 17]byte + var x7535 [1 << 17]byte + var x7536 [1 << 17]byte + var x7537 [1 << 17]byte + var x7538 [1 << 17]byte + var x7539 [1 << 17]byte + var x7540 [1 << 17]byte + var x7541 [1 << 17]byte + var x7542 [1 << 17]byte + var x7543 [1 << 17]byte + var x7544 [1 << 17]byte + var x7545 [1 << 17]byte + var x7546 [1 << 17]byte + var x7547 [1 << 17]byte + var x7548 [1 << 17]byte + var x7549 [1 << 17]byte + var x7550 [1 << 17]byte + var x7551 [1 << 17]byte + var x7552 [1 << 17]byte + var x7553 [1 << 17]byte + var x7554 [1 << 17]byte + var x7555 [1 << 17]byte + var x7556 [1 << 17]byte + var x7557 [1 << 17]byte + var x7558 [1 << 17]byte + var x7559 [1 << 17]byte + var x7560 [1 << 17]byte + var x7561 [1 << 17]byte + var x7562 [1 << 17]byte + var x7563 [1 << 17]byte + var x7564 [1 << 17]byte + var x7565 [1 << 17]byte + var x7566 [1 << 17]byte + var x7567 [1 << 17]byte + var x7568 [1 << 17]byte + var x7569 [1 << 17]byte + var x7570 [1 << 17]byte + var x7571 [1 << 17]byte + var x7572 [1 << 17]byte + var x7573 [1 << 17]byte + var x7574 [1 << 17]byte + var x7575 [1 << 17]byte + var x7576 [1 << 17]byte + var x7577 [1 << 17]byte + var x7578 [1 << 17]byte + var x7579 [1 << 17]byte + var x7580 [1 << 17]byte + var x7581 [1 << 17]byte + var x7582 [1 << 17]byte + var x7583 [1 << 17]byte + var x7584 [1 << 17]byte + var x7585 [1 << 17]byte + var x7586 [1 << 17]byte + var x7587 [1 << 17]byte + var x7588 [1 << 17]byte + var x7589 [1 << 17]byte + var x7590 [1 << 17]byte + var x7591 [1 << 17]byte + var x7592 [1 << 17]byte + var x7593 [1 << 17]byte + var x7594 [1 << 17]byte + var x7595 [1 << 17]byte + var x7596 [1 << 17]byte + var x7597 [1 << 17]byte + var x7598 [1 << 17]byte + var x7599 [1 << 17]byte + var x7600 [1 << 17]byte + var x7601 [1 << 17]byte + var x7602 [1 << 17]byte + var x7603 [1 << 17]byte + var x7604 [1 << 17]byte + var x7605 [1 << 17]byte + var x7606 [1 << 17]byte + var x7607 [1 << 17]byte + var x7608 [1 << 17]byte + var x7609 [1 << 17]byte + var x7610 [1 << 17]byte + var x7611 [1 << 17]byte + var x7612 [1 << 17]byte + var x7613 [1 << 17]byte + var x7614 [1 << 17]byte + var x7615 [1 << 17]byte + var x7616 [1 << 17]byte + var x7617 [1 << 17]byte + var x7618 [1 << 17]byte + var x7619 [1 << 17]byte + var x7620 [1 << 17]byte + var x7621 [1 << 17]byte + var x7622 [1 << 17]byte + var x7623 [1 << 17]byte + var x7624 [1 << 17]byte + var x7625 [1 << 17]byte + var x7626 [1 << 17]byte + var x7627 [1 << 17]byte + var x7628 [1 << 17]byte + var x7629 [1 << 17]byte + var x7630 [1 << 17]byte + var x7631 [1 << 17]byte + var x7632 [1 << 17]byte + var x7633 [1 << 17]byte + var x7634 [1 << 17]byte + var x7635 [1 << 17]byte + var x7636 [1 << 17]byte + var x7637 [1 << 17]byte + var x7638 [1 << 17]byte + var x7639 [1 << 17]byte + var x7640 [1 << 17]byte + var x7641 [1 << 17]byte + var x7642 [1 << 17]byte + var x7643 [1 << 17]byte + var x7644 [1 << 17]byte + var x7645 [1 << 17]byte + var x7646 [1 << 17]byte + var x7647 [1 << 17]byte + var x7648 [1 << 17]byte + var x7649 [1 << 17]byte + var x7650 [1 << 17]byte + var x7651 [1 << 17]byte + var x7652 [1 << 17]byte + var x7653 [1 << 17]byte + var x7654 [1 << 17]byte + var x7655 [1 << 17]byte + var x7656 [1 << 17]byte + var x7657 [1 << 17]byte + var x7658 [1 << 17]byte + var x7659 [1 << 17]byte + var x7660 [1 << 17]byte + var x7661 [1 << 17]byte + var x7662 [1 << 17]byte + var x7663 [1 << 17]byte + var x7664 [1 << 17]byte + var x7665 [1 << 17]byte + var x7666 [1 << 17]byte + var x7667 [1 << 17]byte + var x7668 [1 << 17]byte + var x7669 [1 << 17]byte + var x7670 [1 << 17]byte + var x7671 [1 << 17]byte + var x7672 [1 << 17]byte + var x7673 [1 << 17]byte + var x7674 [1 << 17]byte + var x7675 [1 << 17]byte + var x7676 [1 << 17]byte + var x7677 [1 << 17]byte + var x7678 [1 << 17]byte + var x7679 [1 << 17]byte + var x7680 [1 << 17]byte + var x7681 [1 << 17]byte + var x7682 [1 << 17]byte + var x7683 [1 << 17]byte + var x7684 [1 << 17]byte + var x7685 [1 << 17]byte + var x7686 [1 << 17]byte + var x7687 [1 << 17]byte + var x7688 [1 << 17]byte + var x7689 [1 << 17]byte + var x7690 [1 << 17]byte + var x7691 [1 << 17]byte + var x7692 [1 << 17]byte + var x7693 [1 << 17]byte + var x7694 [1 << 17]byte + var x7695 [1 << 17]byte + var x7696 [1 << 17]byte + var x7697 [1 << 17]byte + var x7698 [1 << 17]byte + var x7699 [1 << 17]byte + var x7700 [1 << 17]byte + var x7701 [1 << 17]byte + var x7702 [1 << 17]byte + var x7703 [1 << 17]byte + var x7704 [1 << 17]byte + var x7705 [1 << 17]byte + var x7706 [1 << 17]byte + var x7707 [1 << 17]byte + var x7708 [1 << 17]byte + var x7709 [1 << 17]byte + var x7710 [1 << 17]byte + var x7711 [1 << 17]byte + var x7712 [1 << 17]byte + var x7713 [1 << 17]byte + var x7714 [1 << 17]byte + var x7715 [1 << 17]byte + var x7716 [1 << 17]byte + var x7717 [1 << 17]byte + var x7718 [1 << 17]byte + var x7719 [1 << 17]byte + var x7720 [1 << 17]byte + var x7721 [1 << 17]byte + var x7722 [1 << 17]byte + var x7723 [1 << 17]byte + var x7724 [1 << 17]byte + var x7725 [1 << 17]byte + var x7726 [1 << 17]byte + var x7727 [1 << 17]byte + var x7728 [1 << 17]byte + var x7729 [1 << 17]byte + var x7730 [1 << 17]byte + var x7731 [1 << 17]byte + var x7732 [1 << 17]byte + var x7733 [1 << 17]byte + var x7734 [1 << 17]byte + var x7735 [1 << 17]byte + var x7736 [1 << 17]byte + var x7737 [1 << 17]byte + var x7738 [1 << 17]byte + var x7739 [1 << 17]byte + var x7740 [1 << 17]byte + var x7741 [1 << 17]byte + var x7742 [1 << 17]byte + var x7743 [1 << 17]byte + var x7744 [1 << 17]byte + var x7745 [1 << 17]byte + var x7746 [1 << 17]byte + var x7747 [1 << 17]byte + var x7748 [1 << 17]byte + var x7749 [1 << 17]byte + var x7750 [1 << 17]byte + var x7751 [1 << 17]byte + var x7752 [1 << 17]byte + var x7753 [1 << 17]byte + var x7754 [1 << 17]byte + var x7755 [1 << 17]byte + var x7756 [1 << 17]byte + var x7757 [1 << 17]byte + var x7758 [1 << 17]byte + var x7759 [1 << 17]byte + var x7760 [1 << 17]byte + var x7761 [1 << 17]byte + var x7762 [1 << 17]byte + var x7763 [1 << 17]byte + var x7764 [1 << 17]byte + var x7765 [1 << 17]byte + var x7766 [1 << 17]byte + var x7767 [1 << 17]byte + var x7768 [1 << 17]byte + var x7769 [1 << 17]byte + var x7770 [1 << 17]byte + var x7771 [1 << 17]byte + var x7772 [1 << 17]byte + var x7773 [1 << 17]byte + var x7774 [1 << 17]byte + var x7775 [1 << 17]byte + var x7776 [1 << 17]byte + var x7777 [1 << 17]byte + var x7778 [1 << 17]byte + var x7779 [1 << 17]byte + var x7780 [1 << 17]byte + var x7781 [1 << 17]byte + var x7782 [1 << 17]byte + var x7783 [1 << 17]byte + var x7784 [1 << 17]byte + var x7785 [1 << 17]byte + var x7786 [1 << 17]byte + var x7787 [1 << 17]byte + var x7788 [1 << 17]byte + var x7789 [1 << 17]byte + var x7790 [1 << 17]byte + var x7791 [1 << 17]byte + var x7792 [1 << 17]byte + var x7793 [1 << 17]byte + var x7794 [1 << 17]byte + var x7795 [1 << 17]byte + var x7796 [1 << 17]byte + var x7797 [1 << 17]byte + var x7798 [1 << 17]byte + var x7799 [1 << 17]byte + var x7800 [1 << 17]byte + var x7801 [1 << 17]byte + var x7802 [1 << 17]byte + var x7803 [1 << 17]byte + var x7804 [1 << 17]byte + var x7805 [1 << 17]byte + var x7806 [1 << 17]byte + var x7807 [1 << 17]byte + var x7808 [1 << 17]byte + var x7809 [1 << 17]byte + var x7810 [1 << 17]byte + var x7811 [1 << 17]byte + var x7812 [1 << 17]byte + var x7813 [1 << 17]byte + var x7814 [1 << 17]byte + var x7815 [1 << 17]byte + var x7816 [1 << 17]byte + var x7817 [1 << 17]byte + var x7818 [1 << 17]byte + var x7819 [1 << 17]byte + var x7820 [1 << 17]byte + var x7821 [1 << 17]byte + var x7822 [1 << 17]byte + var x7823 [1 << 17]byte + var x7824 [1 << 17]byte + var x7825 [1 << 17]byte + var x7826 [1 << 17]byte + var x7827 [1 << 17]byte + var x7828 [1 << 17]byte + var x7829 [1 << 17]byte + var x7830 [1 << 17]byte + var x7831 [1 << 17]byte + var x7832 [1 << 17]byte + var x7833 [1 << 17]byte + var x7834 [1 << 17]byte + var x7835 [1 << 17]byte + var x7836 [1 << 17]byte + var x7837 [1 << 17]byte + var x7838 [1 << 17]byte + var x7839 [1 << 17]byte + var x7840 [1 << 17]byte + var x7841 [1 << 17]byte + var x7842 [1 << 17]byte + var x7843 [1 << 17]byte + var x7844 [1 << 17]byte + var x7845 [1 << 17]byte + var x7846 [1 << 17]byte + var x7847 [1 << 17]byte + var x7848 [1 << 17]byte + var x7849 [1 << 17]byte + var x7850 [1 << 17]byte + var x7851 [1 << 17]byte + var x7852 [1 << 17]byte + var x7853 [1 << 17]byte + var x7854 [1 << 17]byte + var x7855 [1 << 17]byte + var x7856 [1 << 17]byte + var x7857 [1 << 17]byte + var x7858 [1 << 17]byte + var x7859 [1 << 17]byte + var x7860 [1 << 17]byte + var x7861 [1 << 17]byte + var x7862 [1 << 17]byte + var x7863 [1 << 17]byte + var x7864 [1 << 17]byte + var x7865 [1 << 17]byte + var x7866 [1 << 17]byte + var x7867 [1 << 17]byte + var x7868 [1 << 17]byte + var x7869 [1 << 17]byte + var x7870 [1 << 17]byte + var x7871 [1 << 17]byte + var x7872 [1 << 17]byte + var x7873 [1 << 17]byte + var x7874 [1 << 17]byte + var x7875 [1 << 17]byte + var x7876 [1 << 17]byte + var x7877 [1 << 17]byte + var x7878 [1 << 17]byte + var x7879 [1 << 17]byte + var x7880 [1 << 17]byte + var x7881 [1 << 17]byte + var x7882 [1 << 17]byte + var x7883 [1 << 17]byte + var x7884 [1 << 17]byte + var x7885 [1 << 17]byte + var x7886 [1 << 17]byte + var x7887 [1 << 17]byte + var x7888 [1 << 17]byte + var x7889 [1 << 17]byte + var x7890 [1 << 17]byte + var x7891 [1 << 17]byte + var x7892 [1 << 17]byte + var x7893 [1 << 17]byte + var x7894 [1 << 17]byte + var x7895 [1 << 17]byte + var x7896 [1 << 17]byte + var x7897 [1 << 17]byte + var x7898 [1 << 17]byte + var x7899 [1 << 17]byte + var x7900 [1 << 17]byte + var x7901 [1 << 17]byte + var x7902 [1 << 17]byte + var x7903 [1 << 17]byte + var x7904 [1 << 17]byte + var x7905 [1 << 17]byte + var x7906 [1 << 17]byte + var x7907 [1 << 17]byte + var x7908 [1 << 17]byte + var x7909 [1 << 17]byte + var x7910 [1 << 17]byte + var x7911 [1 << 17]byte + var x7912 [1 << 17]byte + var x7913 [1 << 17]byte + var x7914 [1 << 17]byte + var x7915 [1 << 17]byte + var x7916 [1 << 17]byte + var x7917 [1 << 17]byte + var x7918 [1 << 17]byte + var x7919 [1 << 17]byte + var x7920 [1 << 17]byte + var x7921 [1 << 17]byte + var x7922 [1 << 17]byte + var x7923 [1 << 17]byte + var x7924 [1 << 17]byte + var x7925 [1 << 17]byte + var x7926 [1 << 17]byte + var x7927 [1 << 17]byte + var x7928 [1 << 17]byte + var x7929 [1 << 17]byte + var x7930 [1 << 17]byte + var x7931 [1 << 17]byte + var x7932 [1 << 17]byte + var x7933 [1 << 17]byte + var x7934 [1 << 17]byte + var x7935 [1 << 17]byte + var x7936 [1 << 17]byte + var x7937 [1 << 17]byte + var x7938 [1 << 17]byte + var x7939 [1 << 17]byte + var x7940 [1 << 17]byte + var x7941 [1 << 17]byte + var x7942 [1 << 17]byte + var x7943 [1 << 17]byte + var x7944 [1 << 17]byte + var x7945 [1 << 17]byte + var x7946 [1 << 17]byte + var x7947 [1 << 17]byte + var x7948 [1 << 17]byte + var x7949 [1 << 17]byte + var x7950 [1 << 17]byte + var x7951 [1 << 17]byte + var x7952 [1 << 17]byte + var x7953 [1 << 17]byte + var x7954 [1 << 17]byte + var x7955 [1 << 17]byte + var x7956 [1 << 17]byte + var x7957 [1 << 17]byte + var x7958 [1 << 17]byte + var x7959 [1 << 17]byte + var x7960 [1 << 17]byte + var x7961 [1 << 17]byte + var x7962 [1 << 17]byte + var x7963 [1 << 17]byte + var x7964 [1 << 17]byte + var x7965 [1 << 17]byte + var x7966 [1 << 17]byte + var x7967 [1 << 17]byte + var x7968 [1 << 17]byte + var x7969 [1 << 17]byte + var x7970 [1 << 17]byte + var x7971 [1 << 17]byte + var x7972 [1 << 17]byte + var x7973 [1 << 17]byte + var x7974 [1 << 17]byte + var x7975 [1 << 17]byte + var x7976 [1 << 17]byte + var x7977 [1 << 17]byte + var x7978 [1 << 17]byte + var x7979 [1 << 17]byte + var x7980 [1 << 17]byte + var x7981 [1 << 17]byte + var x7982 [1 << 17]byte + var x7983 [1 << 17]byte + var x7984 [1 << 17]byte + var x7985 [1 << 17]byte + var x7986 [1 << 17]byte + var x7987 [1 << 17]byte + var x7988 [1 << 17]byte + var x7989 [1 << 17]byte + var x7990 [1 << 17]byte + var x7991 [1 << 17]byte + var x7992 [1 << 17]byte + var x7993 [1 << 17]byte + var x7994 [1 << 17]byte + var x7995 [1 << 17]byte + var x7996 [1 << 17]byte + var x7997 [1 << 17]byte + var x7998 [1 << 17]byte + var x7999 [1 << 17]byte + var x8000 [1 << 17]byte + var x8001 [1 << 17]byte + var x8002 [1 << 17]byte + var x8003 [1 << 17]byte + var x8004 [1 << 17]byte + var x8005 [1 << 17]byte + var x8006 [1 << 17]byte + var x8007 [1 << 17]byte + var x8008 [1 << 17]byte + var x8009 [1 << 17]byte + var x8010 [1 << 17]byte + var x8011 [1 << 17]byte + var x8012 [1 << 17]byte + var x8013 [1 << 17]byte + var x8014 [1 << 17]byte + var x8015 [1 << 17]byte + var x8016 [1 << 17]byte + var x8017 [1 << 17]byte + var x8018 [1 << 17]byte + var x8019 [1 << 17]byte + var x8020 [1 << 17]byte + var x8021 [1 << 17]byte + var x8022 [1 << 17]byte + var x8023 [1 << 17]byte + var x8024 [1 << 17]byte + var x8025 [1 << 17]byte + var x8026 [1 << 17]byte + var x8027 [1 << 17]byte + var x8028 [1 << 17]byte + var x8029 [1 << 17]byte + var x8030 [1 << 17]byte + var x8031 [1 << 17]byte + var x8032 [1 << 17]byte + var x8033 [1 << 17]byte + var x8034 [1 << 17]byte + var x8035 [1 << 17]byte + var x8036 [1 << 17]byte + var x8037 [1 << 17]byte + var x8038 [1 << 17]byte + var x8039 [1 << 17]byte + var x8040 [1 << 17]byte + var x8041 [1 << 17]byte + var x8042 [1 << 17]byte + var x8043 [1 << 17]byte + var x8044 [1 << 17]byte + var x8045 [1 << 17]byte + var x8046 [1 << 17]byte + var x8047 [1 << 17]byte + var x8048 [1 << 17]byte + var x8049 [1 << 17]byte + var x8050 [1 << 17]byte + var x8051 [1 << 17]byte + var x8052 [1 << 17]byte + var x8053 [1 << 17]byte + var x8054 [1 << 17]byte + var x8055 [1 << 17]byte + var x8056 [1 << 17]byte + var x8057 [1 << 17]byte + var x8058 [1 << 17]byte + var x8059 [1 << 17]byte + var x8060 [1 << 17]byte + var x8061 [1 << 17]byte + var x8062 [1 << 17]byte + var x8063 [1 << 17]byte + var x8064 [1 << 17]byte + var x8065 [1 << 17]byte + var x8066 [1 << 17]byte + var x8067 [1 << 17]byte + var x8068 [1 << 17]byte + var x8069 [1 << 17]byte + var x8070 [1 << 17]byte + var x8071 [1 << 17]byte + var x8072 [1 << 17]byte + var x8073 [1 << 17]byte + var x8074 [1 << 17]byte + var x8075 [1 << 17]byte + var x8076 [1 << 17]byte + var x8077 [1 << 17]byte + var x8078 [1 << 17]byte + var x8079 [1 << 17]byte + var x8080 [1 << 17]byte + var x8081 [1 << 17]byte + var x8082 [1 << 17]byte + var x8083 [1 << 17]byte + var x8084 [1 << 17]byte + var x8085 [1 << 17]byte + var x8086 [1 << 17]byte + var x8087 [1 << 17]byte + var x8088 [1 << 17]byte + var x8089 [1 << 17]byte + var x8090 [1 << 17]byte + var x8091 [1 << 17]byte + var x8092 [1 << 17]byte + var x8093 [1 << 17]byte + var x8094 [1 << 17]byte + var x8095 [1 << 17]byte + var x8096 [1 << 17]byte + var x8097 [1 << 17]byte + var x8098 [1 << 17]byte + var x8099 [1 << 17]byte + var x8100 [1 << 17]byte + var x8101 [1 << 17]byte + var x8102 [1 << 17]byte + var x8103 [1 << 17]byte + var x8104 [1 << 17]byte + var x8105 [1 << 17]byte + var x8106 [1 << 17]byte + var x8107 [1 << 17]byte + var x8108 [1 << 17]byte + var x8109 [1 << 17]byte + var x8110 [1 << 17]byte + var x8111 [1 << 17]byte + var x8112 [1 << 17]byte + var x8113 [1 << 17]byte + var x8114 [1 << 17]byte + var x8115 [1 << 17]byte + var x8116 [1 << 17]byte + var x8117 [1 << 17]byte + var x8118 [1 << 17]byte + var x8119 [1 << 17]byte + var x8120 [1 << 17]byte + var x8121 [1 << 17]byte + var x8122 [1 << 17]byte + var x8123 [1 << 17]byte + var x8124 [1 << 17]byte + var x8125 [1 << 17]byte + var x8126 [1 << 17]byte + var x8127 [1 << 17]byte + var x8128 [1 << 17]byte + var x8129 [1 << 17]byte + var x8130 [1 << 17]byte + var x8131 [1 << 17]byte + var x8132 [1 << 17]byte + var x8133 [1 << 17]byte + var x8134 [1 << 17]byte + var x8135 [1 << 17]byte + var x8136 [1 << 17]byte + var x8137 [1 << 17]byte + var x8138 [1 << 17]byte + var x8139 [1 << 17]byte + var x8140 [1 << 17]byte + var x8141 [1 << 17]byte + var x8142 [1 << 17]byte + var x8143 [1 << 17]byte + var x8144 [1 << 17]byte + var x8145 [1 << 17]byte + var x8146 [1 << 17]byte + var x8147 [1 << 17]byte + var x8148 [1 << 17]byte + var x8149 [1 << 17]byte + var x8150 [1 << 17]byte + var x8151 [1 << 17]byte + var x8152 [1 << 17]byte + var x8153 [1 << 17]byte + var x8154 [1 << 17]byte + var x8155 [1 << 17]byte + var x8156 [1 << 17]byte + var x8157 [1 << 17]byte + var x8158 [1 << 17]byte + var x8159 [1 << 17]byte + var x8160 [1 << 17]byte + var x8161 [1 << 17]byte + var x8162 [1 << 17]byte + var x8163 [1 << 17]byte + var x8164 [1 << 17]byte + var x8165 [1 << 17]byte + var x8166 [1 << 17]byte + var x8167 [1 << 17]byte + var x8168 [1 << 17]byte + var x8169 [1 << 17]byte + var x8170 [1 << 17]byte + var x8171 [1 << 17]byte + var x8172 [1 << 17]byte + var x8173 [1 << 17]byte + var x8174 [1 << 17]byte + var x8175 [1 << 17]byte + var x8176 [1 << 17]byte + var x8177 [1 << 17]byte + var x8178 [1 << 17]byte + var x8179 [1 << 17]byte + var x8180 [1 << 17]byte + var x8181 [1 << 17]byte + var x8182 [1 << 17]byte + var x8183 [1 << 17]byte + var x8184 [1 << 17]byte + var x8185 [1 << 17]byte + var x8186 [1 << 17]byte + var x8187 [1 << 17]byte + var x8188 [1 << 17]byte + var x8189 [1 << 17]byte + var x8190 [1 << 17]byte + var x8191 [1 << 17]byte + var x8192 [1 << 17]byte + var x8193 [1 << 17]byte + var x8194 [1 << 17]byte + var x8195 [1 << 17]byte + var x8196 [1 << 17]byte + var x8197 [1 << 17]byte + var x8198 [1 << 17]byte + var x8199 [1 << 17]byte + var x8200 [1 << 17]byte + var x8201 [1 << 17]byte + var x8202 [1 << 17]byte + var x8203 [1 << 17]byte + var x8204 [1 << 17]byte + var x8205 [1 << 17]byte + var x8206 [1 << 17]byte + var x8207 [1 << 17]byte + var x8208 [1 << 17]byte + var x8209 [1 << 17]byte + var x8210 [1 << 17]byte + var x8211 [1 << 17]byte + var x8212 [1 << 17]byte + var x8213 [1 << 17]byte + var x8214 [1 << 17]byte + var x8215 [1 << 17]byte + var x8216 [1 << 17]byte + var x8217 [1 << 17]byte + var x8218 [1 << 17]byte + var x8219 [1 << 17]byte + var x8220 [1 << 17]byte + var x8221 [1 << 17]byte + var x8222 [1 << 17]byte + var x8223 [1 << 17]byte + var x8224 [1 << 17]byte + var x8225 [1 << 17]byte + var x8226 [1 << 17]byte + var x8227 [1 << 17]byte + var x8228 [1 << 17]byte + var x8229 [1 << 17]byte + var x8230 [1 << 17]byte + var x8231 [1 << 17]byte + var x8232 [1 << 17]byte + var x8233 [1 << 17]byte + var x8234 [1 << 17]byte + var x8235 [1 << 17]byte + var x8236 [1 << 17]byte + var x8237 [1 << 17]byte + var x8238 [1 << 17]byte + var x8239 [1 << 17]byte + var x8240 [1 << 17]byte + var x8241 [1 << 17]byte + var x8242 [1 << 17]byte + var x8243 [1 << 17]byte + var x8244 [1 << 17]byte + var x8245 [1 << 17]byte + var x8246 [1 << 17]byte + var x8247 [1 << 17]byte + var x8248 [1 << 17]byte + var x8249 [1 << 17]byte + var x8250 [1 << 17]byte + var x8251 [1 << 17]byte + var x8252 [1 << 17]byte + var x8253 [1 << 17]byte + var x8254 [1 << 17]byte + var x8255 [1 << 17]byte + var x8256 [1 << 17]byte + var x8257 [1 << 17]byte + var x8258 [1 << 17]byte + var x8259 [1 << 17]byte + var x8260 [1 << 17]byte + var x8261 [1 << 17]byte + var x8262 [1 << 17]byte + var x8263 [1 << 17]byte + var x8264 [1 << 17]byte + var x8265 [1 << 17]byte + var x8266 [1 << 17]byte + var x8267 [1 << 17]byte + var x8268 [1 << 17]byte + var x8269 [1 << 17]byte + var x8270 [1 << 17]byte + var x8271 [1 << 17]byte + var x8272 [1 << 17]byte + var x8273 [1 << 17]byte + var x8274 [1 << 17]byte + var x8275 [1 << 17]byte + var x8276 [1 << 17]byte + var x8277 [1 << 17]byte + var x8278 [1 << 17]byte + var x8279 [1 << 17]byte + var x8280 [1 << 17]byte + var x8281 [1 << 17]byte + var x8282 [1 << 17]byte + var x8283 [1 << 17]byte + var x8284 [1 << 17]byte + var x8285 [1 << 17]byte + var x8286 [1 << 17]byte + var x8287 [1 << 17]byte + var x8288 [1 << 17]byte + var x8289 [1 << 17]byte + var x8290 [1 << 17]byte + var x8291 [1 << 17]byte + var x8292 [1 << 17]byte + var x8293 [1 << 17]byte + var x8294 [1 << 17]byte + var x8295 [1 << 17]byte + var x8296 [1 << 17]byte + var x8297 [1 << 17]byte + var x8298 [1 << 17]byte + var x8299 [1 << 17]byte + var x8300 [1 << 17]byte + var x8301 [1 << 17]byte + var x8302 [1 << 17]byte + var x8303 [1 << 17]byte + var x8304 [1 << 17]byte + var x8305 [1 << 17]byte + var x8306 [1 << 17]byte + var x8307 [1 << 17]byte + var x8308 [1 << 17]byte + var x8309 [1 << 17]byte + var x8310 [1 << 17]byte + var x8311 [1 << 17]byte + var x8312 [1 << 17]byte + var x8313 [1 << 17]byte + var x8314 [1 << 17]byte + var x8315 [1 << 17]byte + var x8316 [1 << 17]byte + var x8317 [1 << 17]byte + var x8318 [1 << 17]byte + var x8319 [1 << 17]byte + var x8320 [1 << 17]byte + var x8321 [1 << 17]byte + var x8322 [1 << 17]byte + var x8323 [1 << 17]byte + var x8324 [1 << 17]byte + var x8325 [1 << 17]byte + var x8326 [1 << 17]byte + var x8327 [1 << 17]byte + var x8328 [1 << 17]byte + var x8329 [1 << 17]byte + var x8330 [1 << 17]byte + var x8331 [1 << 17]byte + var x8332 [1 << 17]byte + var x8333 [1 << 17]byte + var x8334 [1 << 17]byte + var x8335 [1 << 17]byte + var x8336 [1 << 17]byte + var x8337 [1 << 17]byte + var x8338 [1 << 17]byte + var x8339 [1 << 17]byte + var x8340 [1 << 17]byte + var x8341 [1 << 17]byte + var x8342 [1 << 17]byte + var x8343 [1 << 17]byte + var x8344 [1 << 17]byte + var x8345 [1 << 17]byte + var x8346 [1 << 17]byte + var x8347 [1 << 17]byte + var x8348 [1 << 17]byte + var x8349 [1 << 17]byte + var x8350 [1 << 17]byte + var x8351 [1 << 17]byte + var x8352 [1 << 17]byte + var x8353 [1 << 17]byte + var x8354 [1 << 17]byte + var x8355 [1 << 17]byte + var x8356 [1 << 17]byte + var x8357 [1 << 17]byte + var x8358 [1 << 17]byte + var x8359 [1 << 17]byte + var x8360 [1 << 17]byte + var x8361 [1 << 17]byte + var x8362 [1 << 17]byte + var x8363 [1 << 17]byte + var x8364 [1 << 17]byte + var x8365 [1 << 17]byte + var x8366 [1 << 17]byte + var x8367 [1 << 17]byte + var x8368 [1 << 17]byte + var x8369 [1 << 17]byte + var x8370 [1 << 17]byte + var x8371 [1 << 17]byte + var x8372 [1 << 17]byte + var x8373 [1 << 17]byte + var x8374 [1 << 17]byte + var x8375 [1 << 17]byte + var x8376 [1 << 17]byte + var x8377 [1 << 17]byte + var x8378 [1 << 17]byte + var x8379 [1 << 17]byte + var x8380 [1 << 17]byte + var x8381 [1 << 17]byte + var x8382 [1 << 17]byte + var x8383 [1 << 17]byte + var x8384 [1 << 17]byte + var x8385 [1 << 17]byte + var x8386 [1 << 17]byte + var x8387 [1 << 17]byte + var x8388 [1 << 17]byte + var x8389 [1 << 17]byte + var x8390 [1 << 17]byte + var x8391 [1 << 17]byte + var x8392 [1 << 17]byte + var x8393 [1 << 17]byte + var x8394 [1 << 17]byte + var x8395 [1 << 17]byte + var x8396 [1 << 17]byte + var x8397 [1 << 17]byte + var x8398 [1 << 17]byte + var x8399 [1 << 17]byte + var x8400 [1 << 17]byte + var x8401 [1 << 17]byte + var x8402 [1 << 17]byte + var x8403 [1 << 17]byte + var x8404 [1 << 17]byte + var x8405 [1 << 17]byte + var x8406 [1 << 17]byte + var x8407 [1 << 17]byte + var x8408 [1 << 17]byte + var x8409 [1 << 17]byte + var x8410 [1 << 17]byte + var x8411 [1 << 17]byte + var x8412 [1 << 17]byte + var x8413 [1 << 17]byte + var x8414 [1 << 17]byte + var x8415 [1 << 17]byte + var x8416 [1 << 17]byte + var x8417 [1 << 17]byte + var x8418 [1 << 17]byte + var x8419 [1 << 17]byte + var x8420 [1 << 17]byte + var x8421 [1 << 17]byte + var x8422 [1 << 17]byte + var x8423 [1 << 17]byte + var x8424 [1 << 17]byte + var x8425 [1 << 17]byte + var x8426 [1 << 17]byte + var x8427 [1 << 17]byte + var x8428 [1 << 17]byte + var x8429 [1 << 17]byte + var x8430 [1 << 17]byte + var x8431 [1 << 17]byte + var x8432 [1 << 17]byte + var x8433 [1 << 17]byte + var x8434 [1 << 17]byte + var x8435 [1 << 17]byte + var x8436 [1 << 17]byte + var x8437 [1 << 17]byte + var x8438 [1 << 17]byte + var x8439 [1 << 17]byte + var x8440 [1 << 17]byte + var x8441 [1 << 17]byte + var x8442 [1 << 17]byte + var x8443 [1 << 17]byte + var x8444 [1 << 17]byte + var x8445 [1 << 17]byte + var x8446 [1 << 17]byte + var x8447 [1 << 17]byte + var x8448 [1 << 17]byte + var x8449 [1 << 17]byte + var x8450 [1 << 17]byte + var x8451 [1 << 17]byte + var x8452 [1 << 17]byte + var x8453 [1 << 17]byte + var x8454 [1 << 17]byte + var x8455 [1 << 17]byte + var x8456 [1 << 17]byte + var x8457 [1 << 17]byte + var x8458 [1 << 17]byte + var x8459 [1 << 17]byte + var x8460 [1 << 17]byte + var x8461 [1 << 17]byte + var x8462 [1 << 17]byte + var x8463 [1 << 17]byte + var x8464 [1 << 17]byte + var x8465 [1 << 17]byte + var x8466 [1 << 17]byte + var x8467 [1 << 17]byte + var x8468 [1 << 17]byte + var x8469 [1 << 17]byte + var x8470 [1 << 17]byte + var x8471 [1 << 17]byte + var x8472 [1 << 17]byte + var x8473 [1 << 17]byte + var x8474 [1 << 17]byte + var x8475 [1 << 17]byte + var x8476 [1 << 17]byte + var x8477 [1 << 17]byte + var x8478 [1 << 17]byte + var x8479 [1 << 17]byte + var x8480 [1 << 17]byte + var x8481 [1 << 17]byte + var x8482 [1 << 17]byte + var x8483 [1 << 17]byte + var x8484 [1 << 17]byte + var x8485 [1 << 17]byte + var x8486 [1 << 17]byte + var x8487 [1 << 17]byte + var x8488 [1 << 17]byte + var x8489 [1 << 17]byte + var x8490 [1 << 17]byte + var x8491 [1 << 17]byte + var x8492 [1 << 17]byte + var x8493 [1 << 17]byte + var x8494 [1 << 17]byte + var x8495 [1 << 17]byte + var x8496 [1 << 17]byte + var x8497 [1 << 17]byte + var x8498 [1 << 17]byte + var x8499 [1 << 17]byte + var x8500 [1 << 17]byte + var x8501 [1 << 17]byte + var x8502 [1 << 17]byte + var x8503 [1 << 17]byte + var x8504 [1 << 17]byte + var x8505 [1 << 17]byte + var x8506 [1 << 17]byte + var x8507 [1 << 17]byte + var x8508 [1 << 17]byte + var x8509 [1 << 17]byte + var x8510 [1 << 17]byte + var x8511 [1 << 17]byte + var x8512 [1 << 17]byte + var x8513 [1 << 17]byte + var x8514 [1 << 17]byte + var x8515 [1 << 17]byte + var x8516 [1 << 17]byte + var x8517 [1 << 17]byte + var x8518 [1 << 17]byte + var x8519 [1 << 17]byte + var x8520 [1 << 17]byte + var x8521 [1 << 17]byte + var x8522 [1 << 17]byte + var x8523 [1 << 17]byte + var x8524 [1 << 17]byte + var x8525 [1 << 17]byte + var x8526 [1 << 17]byte + var x8527 [1 << 17]byte + var x8528 [1 << 17]byte + var x8529 [1 << 17]byte + var x8530 [1 << 17]byte + var x8531 [1 << 17]byte + var x8532 [1 << 17]byte + var x8533 [1 << 17]byte + var x8534 [1 << 17]byte + var x8535 [1 << 17]byte + var x8536 [1 << 17]byte + var x8537 [1 << 17]byte + var x8538 [1 << 17]byte + var x8539 [1 << 17]byte + var x8540 [1 << 17]byte + var x8541 [1 << 17]byte + var x8542 [1 << 17]byte + var x8543 [1 << 17]byte + var x8544 [1 << 17]byte + var x8545 [1 << 17]byte + var x8546 [1 << 17]byte + var x8547 [1 << 17]byte + var x8548 [1 << 17]byte + var x8549 [1 << 17]byte + var x8550 [1 << 17]byte + var x8551 [1 << 17]byte + var x8552 [1 << 17]byte + var x8553 [1 << 17]byte + var x8554 [1 << 17]byte + var x8555 [1 << 17]byte + var x8556 [1 << 17]byte + var x8557 [1 << 17]byte + var x8558 [1 << 17]byte + var x8559 [1 << 17]byte + var x8560 [1 << 17]byte + var x8561 [1 << 17]byte + var x8562 [1 << 17]byte + var x8563 [1 << 17]byte + var x8564 [1 << 17]byte + var x8565 [1 << 17]byte + var x8566 [1 << 17]byte + var x8567 [1 << 17]byte + var x8568 [1 << 17]byte + var x8569 [1 << 17]byte + var x8570 [1 << 17]byte + var x8571 [1 << 17]byte + var x8572 [1 << 17]byte + var x8573 [1 << 17]byte + var x8574 [1 << 17]byte + var x8575 [1 << 17]byte + var x8576 [1 << 17]byte + var x8577 [1 << 17]byte + var x8578 [1 << 17]byte + var x8579 [1 << 17]byte + var x8580 [1 << 17]byte + var x8581 [1 << 17]byte + var x8582 [1 << 17]byte + var x8583 [1 << 17]byte + var x8584 [1 << 17]byte + var x8585 [1 << 17]byte + var x8586 [1 << 17]byte + var x8587 [1 << 17]byte + var x8588 [1 << 17]byte + var x8589 [1 << 17]byte + var x8590 [1 << 17]byte + var x8591 [1 << 17]byte + var x8592 [1 << 17]byte + var x8593 [1 << 17]byte + var x8594 [1 << 17]byte + var x8595 [1 << 17]byte + var x8596 [1 << 17]byte + var x8597 [1 << 17]byte + var x8598 [1 << 17]byte + var x8599 [1 << 17]byte + var x8600 [1 << 17]byte + var x8601 [1 << 17]byte + var x8602 [1 << 17]byte + var x8603 [1 << 17]byte + var x8604 [1 << 17]byte + var x8605 [1 << 17]byte + var x8606 [1 << 17]byte + var x8607 [1 << 17]byte + var x8608 [1 << 17]byte + var x8609 [1 << 17]byte + var x8610 [1 << 17]byte + var x8611 [1 << 17]byte + var x8612 [1 << 17]byte + var x8613 [1 << 17]byte + var x8614 [1 << 17]byte + var x8615 [1 << 17]byte + var x8616 [1 << 17]byte + var x8617 [1 << 17]byte + var x8618 [1 << 17]byte + var x8619 [1 << 17]byte + var x8620 [1 << 17]byte + var x8621 [1 << 17]byte + var x8622 [1 << 17]byte + var x8623 [1 << 17]byte + var x8624 [1 << 17]byte + var x8625 [1 << 17]byte + var x8626 [1 << 17]byte + var x8627 [1 << 17]byte + var x8628 [1 << 17]byte + var x8629 [1 << 17]byte + var x8630 [1 << 17]byte + var x8631 [1 << 17]byte + var x8632 [1 << 17]byte + var x8633 [1 << 17]byte + var x8634 [1 << 17]byte + var x8635 [1 << 17]byte + var x8636 [1 << 17]byte + var x8637 [1 << 17]byte + var x8638 [1 << 17]byte + var x8639 [1 << 17]byte + var x8640 [1 << 17]byte + var x8641 [1 << 17]byte + var x8642 [1 << 17]byte + var x8643 [1 << 17]byte + var x8644 [1 << 17]byte + var x8645 [1 << 17]byte + var x8646 [1 << 17]byte + var x8647 [1 << 17]byte + var x8648 [1 << 17]byte + var x8649 [1 << 17]byte + var x8650 [1 << 17]byte + var x8651 [1 << 17]byte + var x8652 [1 << 17]byte + var x8653 [1 << 17]byte + var x8654 [1 << 17]byte + var x8655 [1 << 17]byte + var x8656 [1 << 17]byte + var x8657 [1 << 17]byte + var x8658 [1 << 17]byte + var x8659 [1 << 17]byte + var x8660 [1 << 17]byte + var x8661 [1 << 17]byte + var x8662 [1 << 17]byte + var x8663 [1 << 17]byte + var x8664 [1 << 17]byte + var x8665 [1 << 17]byte + var x8666 [1 << 17]byte + var x8667 [1 << 17]byte + var x8668 [1 << 17]byte + var x8669 [1 << 17]byte + var x8670 [1 << 17]byte + var x8671 [1 << 17]byte + var x8672 [1 << 17]byte + var x8673 [1 << 17]byte + var x8674 [1 << 17]byte + var x8675 [1 << 17]byte + var x8676 [1 << 17]byte + var x8677 [1 << 17]byte + var x8678 [1 << 17]byte + var x8679 [1 << 17]byte + var x8680 [1 << 17]byte + var x8681 [1 << 17]byte + var x8682 [1 << 17]byte + var x8683 [1 << 17]byte + var x8684 [1 << 17]byte + var x8685 [1 << 17]byte + var x8686 [1 << 17]byte + var x8687 [1 << 17]byte + var x8688 [1 << 17]byte + var x8689 [1 << 17]byte + var x8690 [1 << 17]byte + var x8691 [1 << 17]byte + var x8692 [1 << 17]byte + var x8693 [1 << 17]byte + var x8694 [1 << 17]byte + var x8695 [1 << 17]byte + var x8696 [1 << 17]byte + var x8697 [1 << 17]byte + var x8698 [1 << 17]byte + var x8699 [1 << 17]byte + var x8700 [1 << 17]byte + var x8701 [1 << 17]byte + var x8702 [1 << 17]byte + var x8703 [1 << 17]byte + var x8704 [1 << 17]byte + var x8705 [1 << 17]byte + var x8706 [1 << 17]byte + var x8707 [1 << 17]byte + var x8708 [1 << 17]byte + var x8709 [1 << 17]byte + var x8710 [1 << 17]byte + var x8711 [1 << 17]byte + var x8712 [1 << 17]byte + var x8713 [1 << 17]byte + var x8714 [1 << 17]byte + var x8715 [1 << 17]byte + var x8716 [1 << 17]byte + var x8717 [1 << 17]byte + var x8718 [1 << 17]byte + var x8719 [1 << 17]byte + var x8720 [1 << 17]byte + var x8721 [1 << 17]byte + var x8722 [1 << 17]byte + var x8723 [1 << 17]byte + var x8724 [1 << 17]byte + var x8725 [1 << 17]byte + var x8726 [1 << 17]byte + var x8727 [1 << 17]byte + var x8728 [1 << 17]byte + var x8729 [1 << 17]byte + var x8730 [1 << 17]byte + var x8731 [1 << 17]byte + var x8732 [1 << 17]byte + var x8733 [1 << 17]byte + var x8734 [1 << 17]byte + var x8735 [1 << 17]byte + var x8736 [1 << 17]byte + var x8737 [1 << 17]byte + var x8738 [1 << 17]byte + var x8739 [1 << 17]byte + var x8740 [1 << 17]byte + var x8741 [1 << 17]byte + var x8742 [1 << 17]byte + var x8743 [1 << 17]byte + var x8744 [1 << 17]byte + var x8745 [1 << 17]byte + var x8746 [1 << 17]byte + var x8747 [1 << 17]byte + var x8748 [1 << 17]byte + var x8749 [1 << 17]byte + var x8750 [1 << 17]byte + var x8751 [1 << 17]byte + var x8752 [1 << 17]byte + var x8753 [1 << 17]byte + var x8754 [1 << 17]byte + var x8755 [1 << 17]byte + var x8756 [1 << 17]byte + var x8757 [1 << 17]byte + var x8758 [1 << 17]byte + var x8759 [1 << 17]byte + var x8760 [1 << 17]byte + var x8761 [1 << 17]byte + var x8762 [1 << 17]byte + var x8763 [1 << 17]byte + var x8764 [1 << 17]byte + var x8765 [1 << 17]byte + var x8766 [1 << 17]byte + var x8767 [1 << 17]byte + var x8768 [1 << 17]byte + var x8769 [1 << 17]byte + var x8770 [1 << 17]byte + var x8771 [1 << 17]byte + var x8772 [1 << 17]byte + var x8773 [1 << 17]byte + var x8774 [1 << 17]byte + var x8775 [1 << 17]byte + var x8776 [1 << 17]byte + var x8777 [1 << 17]byte + var x8778 [1 << 17]byte + var x8779 [1 << 17]byte + var x8780 [1 << 17]byte + var x8781 [1 << 17]byte + var x8782 [1 << 17]byte + var x8783 [1 << 17]byte + var x8784 [1 << 17]byte + var x8785 [1 << 17]byte + var x8786 [1 << 17]byte + var x8787 [1 << 17]byte + var x8788 [1 << 17]byte + var x8789 [1 << 17]byte + var x8790 [1 << 17]byte + var x8791 [1 << 17]byte + var x8792 [1 << 17]byte + var x8793 [1 << 17]byte + var x8794 [1 << 17]byte + var x8795 [1 << 17]byte + var x8796 [1 << 17]byte + var x8797 [1 << 17]byte + var x8798 [1 << 17]byte + var x8799 [1 << 17]byte + var x8800 [1 << 17]byte + var x8801 [1 << 17]byte + var x8802 [1 << 17]byte + var x8803 [1 << 17]byte + var x8804 [1 << 17]byte + var x8805 [1 << 17]byte + var x8806 [1 << 17]byte + var x8807 [1 << 17]byte + var x8808 [1 << 17]byte + var x8809 [1 << 17]byte + var x8810 [1 << 17]byte + var x8811 [1 << 17]byte + var x8812 [1 << 17]byte + var x8813 [1 << 17]byte + var x8814 [1 << 17]byte + var x8815 [1 << 17]byte + var x8816 [1 << 17]byte + var x8817 [1 << 17]byte + var x8818 [1 << 17]byte + var x8819 [1 << 17]byte + var x8820 [1 << 17]byte + var x8821 [1 << 17]byte + var x8822 [1 << 17]byte + var x8823 [1 << 17]byte + var x8824 [1 << 17]byte + var x8825 [1 << 17]byte + var x8826 [1 << 17]byte + var x8827 [1 << 17]byte + var x8828 [1 << 17]byte + var x8829 [1 << 17]byte + var x8830 [1 << 17]byte + var x8831 [1 << 17]byte + var x8832 [1 << 17]byte + var x8833 [1 << 17]byte + var x8834 [1 << 17]byte + var x8835 [1 << 17]byte + var x8836 [1 << 17]byte + var x8837 [1 << 17]byte + var x8838 [1 << 17]byte + var x8839 [1 << 17]byte + var x8840 [1 << 17]byte + var x8841 [1 << 17]byte + var x8842 [1 << 17]byte + var x8843 [1 << 17]byte + var x8844 [1 << 17]byte + var x8845 [1 << 17]byte + var x8846 [1 << 17]byte + var x8847 [1 << 17]byte + var x8848 [1 << 17]byte + var x8849 [1 << 17]byte + var x8850 [1 << 17]byte + var x8851 [1 << 17]byte + var x8852 [1 << 17]byte + var x8853 [1 << 17]byte + var x8854 [1 << 17]byte + var x8855 [1 << 17]byte + var x8856 [1 << 17]byte + var x8857 [1 << 17]byte + var x8858 [1 << 17]byte + var x8859 [1 << 17]byte + var x8860 [1 << 17]byte + var x8861 [1 << 17]byte + var x8862 [1 << 17]byte + var x8863 [1 << 17]byte + var x8864 [1 << 17]byte + var x8865 [1 << 17]byte + var x8866 [1 << 17]byte + var x8867 [1 << 17]byte + var x8868 [1 << 17]byte + var x8869 [1 << 17]byte + var x8870 [1 << 17]byte + var x8871 [1 << 17]byte + var x8872 [1 << 17]byte + var x8873 [1 << 17]byte + var x8874 [1 << 17]byte + var x8875 [1 << 17]byte + var x8876 [1 << 17]byte + var x8877 [1 << 17]byte + var x8878 [1 << 17]byte + var x8879 [1 << 17]byte + var x8880 [1 << 17]byte + var x8881 [1 << 17]byte + var x8882 [1 << 17]byte + var x8883 [1 << 17]byte + var x8884 [1 << 17]byte + var x8885 [1 << 17]byte + var x8886 [1 << 17]byte + var x8887 [1 << 17]byte + var x8888 [1 << 17]byte + var x8889 [1 << 17]byte + var x8890 [1 << 17]byte + var x8891 [1 << 17]byte + var x8892 [1 << 17]byte + var x8893 [1 << 17]byte + var x8894 [1 << 17]byte + var x8895 [1 << 17]byte + var x8896 [1 << 17]byte + var x8897 [1 << 17]byte + var x8898 [1 << 17]byte + var x8899 [1 << 17]byte + var x8900 [1 << 17]byte + var x8901 [1 << 17]byte + var x8902 [1 << 17]byte + var x8903 [1 << 17]byte + var x8904 [1 << 17]byte + var x8905 [1 << 17]byte + var x8906 [1 << 17]byte + var x8907 [1 << 17]byte + var x8908 [1 << 17]byte + var x8909 [1 << 17]byte + var x8910 [1 << 17]byte + var x8911 [1 << 17]byte + var x8912 [1 << 17]byte + var x8913 [1 << 17]byte + var x8914 [1 << 17]byte + var x8915 [1 << 17]byte + var x8916 [1 << 17]byte + var x8917 [1 << 17]byte + var x8918 [1 << 17]byte + var x8919 [1 << 17]byte + var x8920 [1 << 17]byte + var x8921 [1 << 17]byte + var x8922 [1 << 17]byte + var x8923 [1 << 17]byte + var x8924 [1 << 17]byte + var x8925 [1 << 17]byte + var x8926 [1 << 17]byte + var x8927 [1 << 17]byte + var x8928 [1 << 17]byte + var x8929 [1 << 17]byte + var x8930 [1 << 17]byte + var x8931 [1 << 17]byte + var x8932 [1 << 17]byte + var x8933 [1 << 17]byte + var x8934 [1 << 17]byte + var x8935 [1 << 17]byte + var x8936 [1 << 17]byte + var x8937 [1 << 17]byte + var x8938 [1 << 17]byte + var x8939 [1 << 17]byte + var x8940 [1 << 17]byte + var x8941 [1 << 17]byte + var x8942 [1 << 17]byte + var x8943 [1 << 17]byte + var x8944 [1 << 17]byte + var x8945 [1 << 17]byte + var x8946 [1 << 17]byte + var x8947 [1 << 17]byte + var x8948 [1 << 17]byte + var x8949 [1 << 17]byte + var x8950 [1 << 17]byte + var x8951 [1 << 17]byte + var x8952 [1 << 17]byte + var x8953 [1 << 17]byte + var x8954 [1 << 17]byte + var x8955 [1 << 17]byte + var x8956 [1 << 17]byte + var x8957 [1 << 17]byte + var x8958 [1 << 17]byte + var x8959 [1 << 17]byte + var x8960 [1 << 17]byte + var x8961 [1 << 17]byte + var x8962 [1 << 17]byte + var x8963 [1 << 17]byte + var x8964 [1 << 17]byte + var x8965 [1 << 17]byte + var x8966 [1 << 17]byte + var x8967 [1 << 17]byte + var x8968 [1 << 17]byte + var x8969 [1 << 17]byte + var x8970 [1 << 17]byte + var x8971 [1 << 17]byte + var x8972 [1 << 17]byte + var x8973 [1 << 17]byte + var x8974 [1 << 17]byte + var x8975 [1 << 17]byte + var x8976 [1 << 17]byte + var x8977 [1 << 17]byte + var x8978 [1 << 17]byte + var x8979 [1 << 17]byte + var x8980 [1 << 17]byte + var x8981 [1 << 17]byte + var x8982 [1 << 17]byte + var x8983 [1 << 17]byte + var x8984 [1 << 17]byte + var x8985 [1 << 17]byte + var x8986 [1 << 17]byte + var x8987 [1 << 17]byte + var x8988 [1 << 17]byte + var x8989 [1 << 17]byte + var x8990 [1 << 17]byte + var x8991 [1 << 17]byte + var x8992 [1 << 17]byte + var x8993 [1 << 17]byte + var x8994 [1 << 17]byte + var x8995 [1 << 17]byte + var x8996 [1 << 17]byte + var x8997 [1 << 17]byte + var x8998 [1 << 17]byte + var x8999 [1 << 17]byte + var x9000 [1 << 17]byte + var x9001 [1 << 17]byte + var x9002 [1 << 17]byte + var x9003 [1 << 17]byte + var x9004 [1 << 17]byte + var x9005 [1 << 17]byte + var x9006 [1 << 17]byte + var x9007 [1 << 17]byte + var x9008 [1 << 17]byte + var x9009 [1 << 17]byte + var x9010 [1 << 17]byte + var x9011 [1 << 17]byte + var x9012 [1 << 17]byte + var x9013 [1 << 17]byte + var x9014 [1 << 17]byte + var x9015 [1 << 17]byte + var x9016 [1 << 17]byte + var x9017 [1 << 17]byte + var x9018 [1 << 17]byte + var x9019 [1 << 17]byte + var x9020 [1 << 17]byte + var x9021 [1 << 17]byte + var x9022 [1 << 17]byte + var x9023 [1 << 17]byte + var x9024 [1 << 17]byte + var x9025 [1 << 17]byte + var x9026 [1 << 17]byte + var x9027 [1 << 17]byte + var x9028 [1 << 17]byte + var x9029 [1 << 17]byte + var x9030 [1 << 17]byte + var x9031 [1 << 17]byte + var x9032 [1 << 17]byte + var x9033 [1 << 17]byte + var x9034 [1 << 17]byte + var x9035 [1 << 17]byte + var x9036 [1 << 17]byte + var x9037 [1 << 17]byte + var x9038 [1 << 17]byte + var x9039 [1 << 17]byte + var x9040 [1 << 17]byte + var x9041 [1 << 17]byte + var x9042 [1 << 17]byte + var x9043 [1 << 17]byte + var x9044 [1 << 17]byte + var x9045 [1 << 17]byte + var x9046 [1 << 17]byte + var x9047 [1 << 17]byte + var x9048 [1 << 17]byte + var x9049 [1 << 17]byte + var x9050 [1 << 17]byte + var x9051 [1 << 17]byte + var x9052 [1 << 17]byte + var x9053 [1 << 17]byte + var x9054 [1 << 17]byte + var x9055 [1 << 17]byte + var x9056 [1 << 17]byte + var x9057 [1 << 17]byte + var x9058 [1 << 17]byte + var x9059 [1 << 17]byte + var x9060 [1 << 17]byte + var x9061 [1 << 17]byte + var x9062 [1 << 17]byte + var x9063 [1 << 17]byte + var x9064 [1 << 17]byte + var x9065 [1 << 17]byte + var x9066 [1 << 17]byte + var x9067 [1 << 17]byte + var x9068 [1 << 17]byte + var x9069 [1 << 17]byte + var x9070 [1 << 17]byte + var x9071 [1 << 17]byte + var x9072 [1 << 17]byte + var x9073 [1 << 17]byte + var x9074 [1 << 17]byte + var x9075 [1 << 17]byte + var x9076 [1 << 17]byte + var x9077 [1 << 17]byte + var x9078 [1 << 17]byte + var x9079 [1 << 17]byte + var x9080 [1 << 17]byte + var x9081 [1 << 17]byte + var x9082 [1 << 17]byte + var x9083 [1 << 17]byte + var x9084 [1 << 17]byte + var x9085 [1 << 17]byte + var x9086 [1 << 17]byte + var x9087 [1 << 17]byte + var x9088 [1 << 17]byte + var x9089 [1 << 17]byte + var x9090 [1 << 17]byte + var x9091 [1 << 17]byte + var x9092 [1 << 17]byte + var x9093 [1 << 17]byte + var x9094 [1 << 17]byte + var x9095 [1 << 17]byte + var x9096 [1 << 17]byte + var x9097 [1 << 17]byte + var x9098 [1 << 17]byte + var x9099 [1 << 17]byte + var x9100 [1 << 17]byte + var x9101 [1 << 17]byte + var x9102 [1 << 17]byte + var x9103 [1 << 17]byte + var x9104 [1 << 17]byte + var x9105 [1 << 17]byte + var x9106 [1 << 17]byte + var x9107 [1 << 17]byte + var x9108 [1 << 17]byte + var x9109 [1 << 17]byte + var x9110 [1 << 17]byte + var x9111 [1 << 17]byte + var x9112 [1 << 17]byte + var x9113 [1 << 17]byte + var x9114 [1 << 17]byte + var x9115 [1 << 17]byte + var x9116 [1 << 17]byte + var x9117 [1 << 17]byte + var x9118 [1 << 17]byte + var x9119 [1 << 17]byte + var x9120 [1 << 17]byte + var x9121 [1 << 17]byte + var x9122 [1 << 17]byte + var x9123 [1 << 17]byte + var x9124 [1 << 17]byte + var x9125 [1 << 17]byte + var x9126 [1 << 17]byte + var x9127 [1 << 17]byte + var x9128 [1 << 17]byte + var x9129 [1 << 17]byte + var x9130 [1 << 17]byte + var x9131 [1 << 17]byte + var x9132 [1 << 17]byte + var x9133 [1 << 17]byte + var x9134 [1 << 17]byte + var x9135 [1 << 17]byte + var x9136 [1 << 17]byte + var x9137 [1 << 17]byte + var x9138 [1 << 17]byte + var x9139 [1 << 17]byte + var x9140 [1 << 17]byte + var x9141 [1 << 17]byte + var x9142 [1 << 17]byte + var x9143 [1 << 17]byte + var x9144 [1 << 17]byte + var x9145 [1 << 17]byte + var x9146 [1 << 17]byte + var x9147 [1 << 17]byte + var x9148 [1 << 17]byte + var x9149 [1 << 17]byte + var x9150 [1 << 17]byte + var x9151 [1 << 17]byte + var x9152 [1 << 17]byte + var x9153 [1 << 17]byte + var x9154 [1 << 17]byte + var x9155 [1 << 17]byte + var x9156 [1 << 17]byte + var x9157 [1 << 17]byte + var x9158 [1 << 17]byte + var x9159 [1 << 17]byte + var x9160 [1 << 17]byte + var x9161 [1 << 17]byte + var x9162 [1 << 17]byte + var x9163 [1 << 17]byte + var x9164 [1 << 17]byte + var x9165 [1 << 17]byte + var x9166 [1 << 17]byte + var x9167 [1 << 17]byte + var x9168 [1 << 17]byte + var x9169 [1 << 17]byte + var x9170 [1 << 17]byte + var x9171 [1 << 17]byte + var x9172 [1 << 17]byte + var x9173 [1 << 17]byte + var x9174 [1 << 17]byte + var x9175 [1 << 17]byte + var x9176 [1 << 17]byte + var x9177 [1 << 17]byte + var x9178 [1 << 17]byte + var x9179 [1 << 17]byte + var x9180 [1 << 17]byte + var x9181 [1 << 17]byte + var x9182 [1 << 17]byte + var x9183 [1 << 17]byte + var x9184 [1 << 17]byte + var x9185 [1 << 17]byte + var x9186 [1 << 17]byte + var x9187 [1 << 17]byte + var x9188 [1 << 17]byte + var x9189 [1 << 17]byte + var x9190 [1 << 17]byte + var x9191 [1 << 17]byte + var x9192 [1 << 17]byte + var x9193 [1 << 17]byte + var x9194 [1 << 17]byte + var x9195 [1 << 17]byte + var x9196 [1 << 17]byte + var x9197 [1 << 17]byte + var x9198 [1 << 17]byte + var x9199 [1 << 17]byte + var x9200 [1 << 17]byte + var x9201 [1 << 17]byte + var x9202 [1 << 17]byte + var x9203 [1 << 17]byte + var x9204 [1 << 17]byte + var x9205 [1 << 17]byte + var x9206 [1 << 17]byte + var x9207 [1 << 17]byte + var x9208 [1 << 17]byte + var x9209 [1 << 17]byte + var x9210 [1 << 17]byte + var x9211 [1 << 17]byte + var x9212 [1 << 17]byte + var x9213 [1 << 17]byte + var x9214 [1 << 17]byte + var x9215 [1 << 17]byte + var x9216 [1 << 17]byte + var x9217 [1 << 17]byte + var x9218 [1 << 17]byte + var x9219 [1 << 17]byte + var x9220 [1 << 17]byte + var x9221 [1 << 17]byte + var x9222 [1 << 17]byte + var x9223 [1 << 17]byte + var x9224 [1 << 17]byte + var x9225 [1 << 17]byte + var x9226 [1 << 17]byte + var x9227 [1 << 17]byte + var x9228 [1 << 17]byte + var x9229 [1 << 17]byte + var x9230 [1 << 17]byte + var x9231 [1 << 17]byte + var x9232 [1 << 17]byte + var x9233 [1 << 17]byte + var x9234 [1 << 17]byte + var x9235 [1 << 17]byte + var x9236 [1 << 17]byte + var x9237 [1 << 17]byte + var x9238 [1 << 17]byte + var x9239 [1 << 17]byte + var x9240 [1 << 17]byte + var x9241 [1 << 17]byte + var x9242 [1 << 17]byte + var x9243 [1 << 17]byte + var x9244 [1 << 17]byte + var x9245 [1 << 17]byte + var x9246 [1 << 17]byte + var x9247 [1 << 17]byte + var x9248 [1 << 17]byte + var x9249 [1 << 17]byte + var x9250 [1 << 17]byte + var x9251 [1 << 17]byte + var x9252 [1 << 17]byte + var x9253 [1 << 17]byte + var x9254 [1 << 17]byte + var x9255 [1 << 17]byte + var x9256 [1 << 17]byte + var x9257 [1 << 17]byte + var x9258 [1 << 17]byte + var x9259 [1 << 17]byte + var x9260 [1 << 17]byte + var x9261 [1 << 17]byte + var x9262 [1 << 17]byte + var x9263 [1 << 17]byte + var x9264 [1 << 17]byte + var x9265 [1 << 17]byte + var x9266 [1 << 17]byte + var x9267 [1 << 17]byte + var x9268 [1 << 17]byte + var x9269 [1 << 17]byte + var x9270 [1 << 17]byte + var x9271 [1 << 17]byte + var x9272 [1 << 17]byte + var x9273 [1 << 17]byte + var x9274 [1 << 17]byte + var x9275 [1 << 17]byte + var x9276 [1 << 17]byte + var x9277 [1 << 17]byte + var x9278 [1 << 17]byte + var x9279 [1 << 17]byte + var x9280 [1 << 17]byte + var x9281 [1 << 17]byte + var x9282 [1 << 17]byte + var x9283 [1 << 17]byte + var x9284 [1 << 17]byte + var x9285 [1 << 17]byte + var x9286 [1 << 17]byte + var x9287 [1 << 17]byte + var x9288 [1 << 17]byte + var x9289 [1 << 17]byte + var x9290 [1 << 17]byte + var x9291 [1 << 17]byte + var x9292 [1 << 17]byte + var x9293 [1 << 17]byte + var x9294 [1 << 17]byte + var x9295 [1 << 17]byte + var x9296 [1 << 17]byte + var x9297 [1 << 17]byte + var x9298 [1 << 17]byte + var x9299 [1 << 17]byte + var x9300 [1 << 17]byte + var x9301 [1 << 17]byte + var x9302 [1 << 17]byte + var x9303 [1 << 17]byte + var x9304 [1 << 17]byte + var x9305 [1 << 17]byte + var x9306 [1 << 17]byte + var x9307 [1 << 17]byte + var x9308 [1 << 17]byte + var x9309 [1 << 17]byte + var x9310 [1 << 17]byte + var x9311 [1 << 17]byte + var x9312 [1 << 17]byte + var x9313 [1 << 17]byte + var x9314 [1 << 17]byte + var x9315 [1 << 17]byte + var x9316 [1 << 17]byte + var x9317 [1 << 17]byte + var x9318 [1 << 17]byte + var x9319 [1 << 17]byte + var x9320 [1 << 17]byte + var x9321 [1 << 17]byte + var x9322 [1 << 17]byte + var x9323 [1 << 17]byte + var x9324 [1 << 17]byte + var x9325 [1 << 17]byte + var x9326 [1 << 17]byte + var x9327 [1 << 17]byte + var x9328 [1 << 17]byte + var x9329 [1 << 17]byte + var x9330 [1 << 17]byte + var x9331 [1 << 17]byte + var x9332 [1 << 17]byte + var x9333 [1 << 17]byte + var x9334 [1 << 17]byte + var x9335 [1 << 17]byte + var x9336 [1 << 17]byte + var x9337 [1 << 17]byte + var x9338 [1 << 17]byte + var x9339 [1 << 17]byte + var x9340 [1 << 17]byte + var x9341 [1 << 17]byte + var x9342 [1 << 17]byte + var x9343 [1 << 17]byte + var x9344 [1 << 17]byte + var x9345 [1 << 17]byte + var x9346 [1 << 17]byte + var x9347 [1 << 17]byte + var x9348 [1 << 17]byte + var x9349 [1 << 17]byte + var x9350 [1 << 17]byte + var x9351 [1 << 17]byte + var x9352 [1 << 17]byte + var x9353 [1 << 17]byte + var x9354 [1 << 17]byte + var x9355 [1 << 17]byte + var x9356 [1 << 17]byte + var x9357 [1 << 17]byte + var x9358 [1 << 17]byte + var x9359 [1 << 17]byte + var x9360 [1 << 17]byte + var x9361 [1 << 17]byte + var x9362 [1 << 17]byte + var x9363 [1 << 17]byte + var x9364 [1 << 17]byte + var x9365 [1 << 17]byte + var x9366 [1 << 17]byte + var x9367 [1 << 17]byte + var x9368 [1 << 17]byte + var x9369 [1 << 17]byte + var x9370 [1 << 17]byte + var x9371 [1 << 17]byte + var x9372 [1 << 17]byte + var x9373 [1 << 17]byte + var x9374 [1 << 17]byte + var x9375 [1 << 17]byte + var x9376 [1 << 17]byte + var x9377 [1 << 17]byte + var x9378 [1 << 17]byte + var x9379 [1 << 17]byte + var x9380 [1 << 17]byte + var x9381 [1 << 17]byte + var x9382 [1 << 17]byte + var x9383 [1 << 17]byte + var x9384 [1 << 17]byte + var x9385 [1 << 17]byte + var x9386 [1 << 17]byte + var x9387 [1 << 17]byte + var x9388 [1 << 17]byte + var x9389 [1 << 17]byte + var x9390 [1 << 17]byte + var x9391 [1 << 17]byte + var x9392 [1 << 17]byte + var x9393 [1 << 17]byte + var x9394 [1 << 17]byte + var x9395 [1 << 17]byte + var x9396 [1 << 17]byte + var x9397 [1 << 17]byte + var x9398 [1 << 17]byte + var x9399 [1 << 17]byte + var x9400 [1 << 17]byte + var x9401 [1 << 17]byte + var x9402 [1 << 17]byte + var x9403 [1 << 17]byte + var x9404 [1 << 17]byte + var x9405 [1 << 17]byte + var x9406 [1 << 17]byte + var x9407 [1 << 17]byte + var x9408 [1 << 17]byte + var x9409 [1 << 17]byte + var x9410 [1 << 17]byte + var x9411 [1 << 17]byte + var x9412 [1 << 17]byte + var x9413 [1 << 17]byte + var x9414 [1 << 17]byte + var x9415 [1 << 17]byte + var x9416 [1 << 17]byte + var x9417 [1 << 17]byte + var x9418 [1 << 17]byte + var x9419 [1 << 17]byte + var x9420 [1 << 17]byte + var x9421 [1 << 17]byte + var x9422 [1 << 17]byte + var x9423 [1 << 17]byte + var x9424 [1 << 17]byte + var x9425 [1 << 17]byte + var x9426 [1 << 17]byte + var x9427 [1 << 17]byte + var x9428 [1 << 17]byte + var x9429 [1 << 17]byte + var x9430 [1 << 17]byte + var x9431 [1 << 17]byte + var x9432 [1 << 17]byte + var x9433 [1 << 17]byte + var x9434 [1 << 17]byte + var x9435 [1 << 17]byte + var x9436 [1 << 17]byte + var x9437 [1 << 17]byte + var x9438 [1 << 17]byte + var x9439 [1 << 17]byte + var x9440 [1 << 17]byte + var x9441 [1 << 17]byte + var x9442 [1 << 17]byte + var x9443 [1 << 17]byte + var x9444 [1 << 17]byte + var x9445 [1 << 17]byte + var x9446 [1 << 17]byte + var x9447 [1 << 17]byte + var x9448 [1 << 17]byte + var x9449 [1 << 17]byte + var x9450 [1 << 17]byte + var x9451 [1 << 17]byte + var x9452 [1 << 17]byte + var x9453 [1 << 17]byte + var x9454 [1 << 17]byte + var x9455 [1 << 17]byte + var x9456 [1 << 17]byte + var x9457 [1 << 17]byte + var x9458 [1 << 17]byte + var x9459 [1 << 17]byte + var x9460 [1 << 17]byte + var x9461 [1 << 17]byte + var x9462 [1 << 17]byte + var x9463 [1 << 17]byte + var x9464 [1 << 17]byte + var x9465 [1 << 17]byte + var x9466 [1 << 17]byte + var x9467 [1 << 17]byte + var x9468 [1 << 17]byte + var x9469 [1 << 17]byte + var x9470 [1 << 17]byte + var x9471 [1 << 17]byte + var x9472 [1 << 17]byte + var x9473 [1 << 17]byte + var x9474 [1 << 17]byte + var x9475 [1 << 17]byte + var x9476 [1 << 17]byte + var x9477 [1 << 17]byte + var x9478 [1 << 17]byte + var x9479 [1 << 17]byte + var x9480 [1 << 17]byte + var x9481 [1 << 17]byte + var x9482 [1 << 17]byte + var x9483 [1 << 17]byte + var x9484 [1 << 17]byte + var x9485 [1 << 17]byte + var x9486 [1 << 17]byte + var x9487 [1 << 17]byte + var x9488 [1 << 17]byte + var x9489 [1 << 17]byte + var x9490 [1 << 17]byte + var x9491 [1 << 17]byte + var x9492 [1 << 17]byte + var x9493 [1 << 17]byte + var x9494 [1 << 17]byte + var x9495 [1 << 17]byte + var x9496 [1 << 17]byte + var x9497 [1 << 17]byte + var x9498 [1 << 17]byte + var x9499 [1 << 17]byte + var x9500 [1 << 17]byte + var x9501 [1 << 17]byte + var x9502 [1 << 17]byte + var x9503 [1 << 17]byte + var x9504 [1 << 17]byte + var x9505 [1 << 17]byte + var x9506 [1 << 17]byte + var x9507 [1 << 17]byte + var x9508 [1 << 17]byte + var x9509 [1 << 17]byte + var x9510 [1 << 17]byte + var x9511 [1 << 17]byte + var x9512 [1 << 17]byte + var x9513 [1 << 17]byte + var x9514 [1 << 17]byte + var x9515 [1 << 17]byte + var x9516 [1 << 17]byte + var x9517 [1 << 17]byte + var x9518 [1 << 17]byte + var x9519 [1 << 17]byte + var x9520 [1 << 17]byte + var x9521 [1 << 17]byte + var x9522 [1 << 17]byte + var x9523 [1 << 17]byte + var x9524 [1 << 17]byte + var x9525 [1 << 17]byte + var x9526 [1 << 17]byte + var x9527 [1 << 17]byte + var x9528 [1 << 17]byte + var x9529 [1 << 17]byte + var x9530 [1 << 17]byte + var x9531 [1 << 17]byte + var x9532 [1 << 17]byte + var x9533 [1 << 17]byte + var x9534 [1 << 17]byte + var x9535 [1 << 17]byte + var x9536 [1 << 17]byte + var x9537 [1 << 17]byte + var x9538 [1 << 17]byte + var x9539 [1 << 17]byte + var x9540 [1 << 17]byte + var x9541 [1 << 17]byte + var x9542 [1 << 17]byte + var x9543 [1 << 17]byte + var x9544 [1 << 17]byte + var x9545 [1 << 17]byte + var x9546 [1 << 17]byte + var x9547 [1 << 17]byte + var x9548 [1 << 17]byte + var x9549 [1 << 17]byte + var x9550 [1 << 17]byte + var x9551 [1 << 17]byte + var x9552 [1 << 17]byte + var x9553 [1 << 17]byte + var x9554 [1 << 17]byte + var x9555 [1 << 17]byte + var x9556 [1 << 17]byte + var x9557 [1 << 17]byte + var x9558 [1 << 17]byte + var x9559 [1 << 17]byte + var x9560 [1 << 17]byte + var x9561 [1 << 17]byte + var x9562 [1 << 17]byte + var x9563 [1 << 17]byte + var x9564 [1 << 17]byte + var x9565 [1 << 17]byte + var x9566 [1 << 17]byte + var x9567 [1 << 17]byte + var x9568 [1 << 17]byte + var x9569 [1 << 17]byte + var x9570 [1 << 17]byte + var x9571 [1 << 17]byte + var x9572 [1 << 17]byte + var x9573 [1 << 17]byte + var x9574 [1 << 17]byte + var x9575 [1 << 17]byte + var x9576 [1 << 17]byte + var x9577 [1 << 17]byte + var x9578 [1 << 17]byte + var x9579 [1 << 17]byte + var x9580 [1 << 17]byte + var x9581 [1 << 17]byte + var x9582 [1 << 17]byte + var x9583 [1 << 17]byte + var x9584 [1 << 17]byte + var x9585 [1 << 17]byte + var x9586 [1 << 17]byte + var x9587 [1 << 17]byte + var x9588 [1 << 17]byte + var x9589 [1 << 17]byte + var x9590 [1 << 17]byte + var x9591 [1 << 17]byte + var x9592 [1 << 17]byte + var x9593 [1 << 17]byte + var x9594 [1 << 17]byte + var x9595 [1 << 17]byte + var x9596 [1 << 17]byte + var x9597 [1 << 17]byte + var x9598 [1 << 17]byte + var x9599 [1 << 17]byte + var x9600 [1 << 17]byte + var x9601 [1 << 17]byte + var x9602 [1 << 17]byte + var x9603 [1 << 17]byte + var x9604 [1 << 17]byte + var x9605 [1 << 17]byte + var x9606 [1 << 17]byte + var x9607 [1 << 17]byte + var x9608 [1 << 17]byte + var x9609 [1 << 17]byte + var x9610 [1 << 17]byte + var x9611 [1 << 17]byte + var x9612 [1 << 17]byte + var x9613 [1 << 17]byte + var x9614 [1 << 17]byte + var x9615 [1 << 17]byte + var x9616 [1 << 17]byte + var x9617 [1 << 17]byte + var x9618 [1 << 17]byte + var x9619 [1 << 17]byte + var x9620 [1 << 17]byte + var x9621 [1 << 17]byte + var x9622 [1 << 17]byte + var x9623 [1 << 17]byte + var x9624 [1 << 17]byte + var x9625 [1 << 17]byte + var x9626 [1 << 17]byte + var x9627 [1 << 17]byte + var x9628 [1 << 17]byte + var x9629 [1 << 17]byte + var x9630 [1 << 17]byte + var x9631 [1 << 17]byte + var x9632 [1 << 17]byte + var x9633 [1 << 17]byte + var x9634 [1 << 17]byte + var x9635 [1 << 17]byte + var x9636 [1 << 17]byte + var x9637 [1 << 17]byte + var x9638 [1 << 17]byte + var x9639 [1 << 17]byte + var x9640 [1 << 17]byte + var x9641 [1 << 17]byte + var x9642 [1 << 17]byte + var x9643 [1 << 17]byte + var x9644 [1 << 17]byte + var x9645 [1 << 17]byte + var x9646 [1 << 17]byte + var x9647 [1 << 17]byte + var x9648 [1 << 17]byte + var x9649 [1 << 17]byte + var x9650 [1 << 17]byte + var x9651 [1 << 17]byte + var x9652 [1 << 17]byte + var x9653 [1 << 17]byte + var x9654 [1 << 17]byte + var x9655 [1 << 17]byte + var x9656 [1 << 17]byte + var x9657 [1 << 17]byte + var x9658 [1 << 17]byte + var x9659 [1 << 17]byte + var x9660 [1 << 17]byte + var x9661 [1 << 17]byte + var x9662 [1 << 17]byte + var x9663 [1 << 17]byte + var x9664 [1 << 17]byte + var x9665 [1 << 17]byte + var x9666 [1 << 17]byte + var x9667 [1 << 17]byte + var x9668 [1 << 17]byte + var x9669 [1 << 17]byte + var x9670 [1 << 17]byte + var x9671 [1 << 17]byte + var x9672 [1 << 17]byte + var x9673 [1 << 17]byte + var x9674 [1 << 17]byte + var x9675 [1 << 17]byte + var x9676 [1 << 17]byte + var x9677 [1 << 17]byte + var x9678 [1 << 17]byte + var x9679 [1 << 17]byte + var x9680 [1 << 17]byte + var x9681 [1 << 17]byte + var x9682 [1 << 17]byte + var x9683 [1 << 17]byte + var x9684 [1 << 17]byte + var x9685 [1 << 17]byte + var x9686 [1 << 17]byte + var x9687 [1 << 17]byte + var x9688 [1 << 17]byte + var x9689 [1 << 17]byte + var x9690 [1 << 17]byte + var x9691 [1 << 17]byte + var x9692 [1 << 17]byte + var x9693 [1 << 17]byte + var x9694 [1 << 17]byte + var x9695 [1 << 17]byte + var x9696 [1 << 17]byte + var x9697 [1 << 17]byte + var x9698 [1 << 17]byte + var x9699 [1 << 17]byte + var x9700 [1 << 17]byte + var x9701 [1 << 17]byte + var x9702 [1 << 17]byte + var x9703 [1 << 17]byte + var x9704 [1 << 17]byte + var x9705 [1 << 17]byte + var x9706 [1 << 17]byte + var x9707 [1 << 17]byte + var x9708 [1 << 17]byte + var x9709 [1 << 17]byte + var x9710 [1 << 17]byte + var x9711 [1 << 17]byte + var x9712 [1 << 17]byte + var x9713 [1 << 17]byte + var x9714 [1 << 17]byte + var x9715 [1 << 17]byte + var x9716 [1 << 17]byte + var x9717 [1 << 17]byte + var x9718 [1 << 17]byte + var x9719 [1 << 17]byte + var x9720 [1 << 17]byte + var x9721 [1 << 17]byte + var x9722 [1 << 17]byte + var x9723 [1 << 17]byte + var x9724 [1 << 17]byte + var x9725 [1 << 17]byte + var x9726 [1 << 17]byte + var x9727 [1 << 17]byte + var x9728 [1 << 17]byte + var x9729 [1 << 17]byte + var x9730 [1 << 17]byte + var x9731 [1 << 17]byte + var x9732 [1 << 17]byte + var x9733 [1 << 17]byte + var x9734 [1 << 17]byte + var x9735 [1 << 17]byte + var x9736 [1 << 17]byte + var x9737 [1 << 17]byte + var x9738 [1 << 17]byte + var x9739 [1 << 17]byte + var x9740 [1 << 17]byte + var x9741 [1 << 17]byte + var x9742 [1 << 17]byte + var x9743 [1 << 17]byte + var x9744 [1 << 17]byte + var x9745 [1 << 17]byte + var x9746 [1 << 17]byte + var x9747 [1 << 17]byte + var x9748 [1 << 17]byte + var x9749 [1 << 17]byte + var x9750 [1 << 17]byte + var x9751 [1 << 17]byte + var x9752 [1 << 17]byte + var x9753 [1 << 17]byte + var x9754 [1 << 17]byte + var x9755 [1 << 17]byte + var x9756 [1 << 17]byte + var x9757 [1 << 17]byte + var x9758 [1 << 17]byte + var x9759 [1 << 17]byte + var x9760 [1 << 17]byte + var x9761 [1 << 17]byte + var x9762 [1 << 17]byte + var x9763 [1 << 17]byte + var x9764 [1 << 17]byte + var x9765 [1 << 17]byte + var x9766 [1 << 17]byte + var x9767 [1 << 17]byte + var x9768 [1 << 17]byte + var x9769 [1 << 17]byte + var x9770 [1 << 17]byte + var x9771 [1 << 17]byte + var x9772 [1 << 17]byte + var x9773 [1 << 17]byte + var x9774 [1 << 17]byte + var x9775 [1 << 17]byte + var x9776 [1 << 17]byte + var x9777 [1 << 17]byte + var x9778 [1 << 17]byte + var x9779 [1 << 17]byte + var x9780 [1 << 17]byte + var x9781 [1 << 17]byte + var x9782 [1 << 17]byte + var x9783 [1 << 17]byte + var x9784 [1 << 17]byte + var x9785 [1 << 17]byte + var x9786 [1 << 17]byte + var x9787 [1 << 17]byte + var x9788 [1 << 17]byte + var x9789 [1 << 17]byte + var x9790 [1 << 17]byte + var x9791 [1 << 17]byte + var x9792 [1 << 17]byte + var x9793 [1 << 17]byte + var x9794 [1 << 17]byte + var x9795 [1 << 17]byte + var x9796 [1 << 17]byte + var x9797 [1 << 17]byte + var x9798 [1 << 17]byte + var x9799 [1 << 17]byte + var x9800 [1 << 17]byte + var x9801 [1 << 17]byte + var x9802 [1 << 17]byte + var x9803 [1 << 17]byte + var x9804 [1 << 17]byte + var x9805 [1 << 17]byte + var x9806 [1 << 17]byte + var x9807 [1 << 17]byte + var x9808 [1 << 17]byte + var x9809 [1 << 17]byte + var x9810 [1 << 17]byte + var x9811 [1 << 17]byte + var x9812 [1 << 17]byte + var x9813 [1 << 17]byte + var x9814 [1 << 17]byte + var x9815 [1 << 17]byte + var x9816 [1 << 17]byte + var x9817 [1 << 17]byte + var x9818 [1 << 17]byte + var x9819 [1 << 17]byte + var x9820 [1 << 17]byte + var x9821 [1 << 17]byte + var x9822 [1 << 17]byte + var x9823 [1 << 17]byte + var x9824 [1 << 17]byte + var x9825 [1 << 17]byte + var x9826 [1 << 17]byte + var x9827 [1 << 17]byte + var x9828 [1 << 17]byte + var x9829 [1 << 17]byte + var x9830 [1 << 17]byte + var x9831 [1 << 17]byte + var x9832 [1 << 17]byte + var x9833 [1 << 17]byte + var x9834 [1 << 17]byte + var x9835 [1 << 17]byte + var x9836 [1 << 17]byte + var x9837 [1 << 17]byte + var x9838 [1 << 17]byte + var x9839 [1 << 17]byte + var x9840 [1 << 17]byte + var x9841 [1 << 17]byte + var x9842 [1 << 17]byte + var x9843 [1 << 17]byte + var x9844 [1 << 17]byte + var x9845 [1 << 17]byte + var x9846 [1 << 17]byte + var x9847 [1 << 17]byte + var x9848 [1 << 17]byte + var x9849 [1 << 17]byte + var x9850 [1 << 17]byte + var x9851 [1 << 17]byte + var x9852 [1 << 17]byte + var x9853 [1 << 17]byte + var x9854 [1 << 17]byte + var x9855 [1 << 17]byte + var x9856 [1 << 17]byte + var x9857 [1 << 17]byte + var x9858 [1 << 17]byte + var x9859 [1 << 17]byte + var x9860 [1 << 17]byte + var x9861 [1 << 17]byte + var x9862 [1 << 17]byte + var x9863 [1 << 17]byte + var x9864 [1 << 17]byte + var x9865 [1 << 17]byte + var x9866 [1 << 17]byte + var x9867 [1 << 17]byte + var x9868 [1 << 17]byte + var x9869 [1 << 17]byte + var x9870 [1 << 17]byte + var x9871 [1 << 17]byte + var x9872 [1 << 17]byte + var x9873 [1 << 17]byte + var x9874 [1 << 17]byte + var x9875 [1 << 17]byte + var x9876 [1 << 17]byte + var x9877 [1 << 17]byte + var x9878 [1 << 17]byte + var x9879 [1 << 17]byte + var x9880 [1 << 17]byte + var x9881 [1 << 17]byte + var x9882 [1 << 17]byte + var x9883 [1 << 17]byte + var x9884 [1 << 17]byte + var x9885 [1 << 17]byte + var x9886 [1 << 17]byte + var x9887 [1 << 17]byte + var x9888 [1 << 17]byte + var x9889 [1 << 17]byte + var x9890 [1 << 17]byte + var x9891 [1 << 17]byte + var x9892 [1 << 17]byte + var x9893 [1 << 17]byte + var x9894 [1 << 17]byte + var x9895 [1 << 17]byte + var x9896 [1 << 17]byte + var x9897 [1 << 17]byte + var x9898 [1 << 17]byte + var x9899 [1 << 17]byte + var x9900 [1 << 17]byte + var x9901 [1 << 17]byte + var x9902 [1 << 17]byte + var x9903 [1 << 17]byte + var x9904 [1 << 17]byte + var x9905 [1 << 17]byte + var x9906 [1 << 17]byte + var x9907 [1 << 17]byte + var x9908 [1 << 17]byte + var x9909 [1 << 17]byte + var x9910 [1 << 17]byte + var x9911 [1 << 17]byte + var x9912 [1 << 17]byte + var x9913 [1 << 17]byte + var x9914 [1 << 17]byte + var x9915 [1 << 17]byte + var x9916 [1 << 17]byte + var x9917 [1 << 17]byte + var x9918 [1 << 17]byte + var x9919 [1 << 17]byte + var x9920 [1 << 17]byte + var x9921 [1 << 17]byte + var x9922 [1 << 17]byte + var x9923 [1 << 17]byte + var x9924 [1 << 17]byte + var x9925 [1 << 17]byte + var x9926 [1 << 17]byte + var x9927 [1 << 17]byte + var x9928 [1 << 17]byte + var x9929 [1 << 17]byte + var x9930 [1 << 17]byte + var x9931 [1 << 17]byte + var x9932 [1 << 17]byte + var x9933 [1 << 17]byte + var x9934 [1 << 17]byte + var x9935 [1 << 17]byte + var x9936 [1 << 17]byte + var x9937 [1 << 17]byte + var x9938 [1 << 17]byte + var x9939 [1 << 17]byte + var x9940 [1 << 17]byte + var x9941 [1 << 17]byte + var x9942 [1 << 17]byte + var x9943 [1 << 17]byte + var x9944 [1 << 17]byte + var x9945 [1 << 17]byte + var x9946 [1 << 17]byte + var x9947 [1 << 17]byte + var x9948 [1 << 17]byte + var x9949 [1 << 17]byte + var x9950 [1 << 17]byte + var x9951 [1 << 17]byte + var x9952 [1 << 17]byte + var x9953 [1 << 17]byte + var x9954 [1 << 17]byte + var x9955 [1 << 17]byte + var x9956 [1 << 17]byte + var x9957 [1 << 17]byte + var x9958 [1 << 17]byte + var x9959 [1 << 17]byte + var x9960 [1 << 17]byte + var x9961 [1 << 17]byte + var x9962 [1 << 17]byte + var x9963 [1 << 17]byte + var x9964 [1 << 17]byte + var x9965 [1 << 17]byte + var x9966 [1 << 17]byte + var x9967 [1 << 17]byte + var x9968 [1 << 17]byte + var x9969 [1 << 17]byte + var x9970 [1 << 17]byte + var x9971 [1 << 17]byte + var x9972 [1 << 17]byte + var x9973 [1 << 17]byte + var x9974 [1 << 17]byte + var x9975 [1 << 17]byte + var x9976 [1 << 17]byte + var x9977 [1 << 17]byte + var x9978 [1 << 17]byte + var x9979 [1 << 17]byte + var x9980 [1 << 17]byte + var x9981 [1 << 17]byte + var x9982 [1 << 17]byte + var x9983 [1 << 17]byte + var x9984 [1 << 17]byte + var x9985 [1 << 17]byte + var x9986 [1 << 17]byte + var x9987 [1 << 17]byte + var x9988 [1 << 17]byte + var x9989 [1 << 17]byte + var x9990 [1 << 17]byte + var x9991 [1 << 17]byte + var x9992 [1 << 17]byte + var x9993 [1 << 17]byte + var x9994 [1 << 17]byte + var x9995 [1 << 17]byte + var x9996 [1 << 17]byte + var x9997 [1 << 17]byte + var x9998 [1 << 17]byte + var x9999 [1 << 17]byte + var x10000 [1 << 17]byte + var x10001 [1 << 17]byte + var x10002 [1 << 17]byte + var x10003 [1 << 17]byte + var x10004 [1 << 17]byte + var x10005 [1 << 17]byte + var x10006 [1 << 17]byte + var x10007 [1 << 17]byte + var x10008 [1 << 17]byte + var x10009 [1 << 17]byte + var x10010 [1 << 17]byte + var x10011 [1 << 17]byte + var x10012 [1 << 17]byte + var x10013 [1 << 17]byte + var x10014 [1 << 17]byte + var x10015 [1 << 17]byte + var x10016 [1 << 17]byte + var x10017 [1 << 17]byte + var x10018 [1 << 17]byte + var x10019 [1 << 17]byte + var x10020 [1 << 17]byte + var x10021 [1 << 17]byte + var x10022 [1 << 17]byte + var x10023 [1 << 17]byte + var x10024 [1 << 17]byte + var x10025 [1 << 17]byte + var x10026 [1 << 17]byte + var x10027 [1 << 17]byte + var x10028 [1 << 17]byte + var x10029 [1 << 17]byte + var x10030 [1 << 17]byte + var x10031 [1 << 17]byte + var x10032 [1 << 17]byte + var x10033 [1 << 17]byte + var x10034 [1 << 17]byte + var x10035 [1 << 17]byte + var x10036 [1 << 17]byte + var x10037 [1 << 17]byte + var x10038 [1 << 17]byte + var x10039 [1 << 17]byte + var x10040 [1 << 17]byte + var x10041 [1 << 17]byte + var x10042 [1 << 17]byte + var x10043 [1 << 17]byte + var x10044 [1 << 17]byte + var x10045 [1 << 17]byte + var x10046 [1 << 17]byte + var x10047 [1 << 17]byte + var x10048 [1 << 17]byte + var x10049 [1 << 17]byte + var x10050 [1 << 17]byte + var x10051 [1 << 17]byte + var x10052 [1 << 17]byte + var x10053 [1 << 17]byte + var x10054 [1 << 17]byte + var x10055 [1 << 17]byte + var x10056 [1 << 17]byte + var x10057 [1 << 17]byte + var x10058 [1 << 17]byte + var x10059 [1 << 17]byte + var x10060 [1 << 17]byte + var x10061 [1 << 17]byte + var x10062 [1 << 17]byte + var x10063 [1 << 17]byte + var x10064 [1 << 17]byte + var x10065 [1 << 17]byte + var x10066 [1 << 17]byte + var x10067 [1 << 17]byte + var x10068 [1 << 17]byte + var x10069 [1 << 17]byte + var x10070 [1 << 17]byte + var x10071 [1 << 17]byte + var x10072 [1 << 17]byte + var x10073 [1 << 17]byte + var x10074 [1 << 17]byte + var x10075 [1 << 17]byte + var x10076 [1 << 17]byte + var x10077 [1 << 17]byte + var x10078 [1 << 17]byte + var x10079 [1 << 17]byte + var x10080 [1 << 17]byte + var x10081 [1 << 17]byte + var x10082 [1 << 17]byte + var x10083 [1 << 17]byte + var x10084 [1 << 17]byte + var x10085 [1 << 17]byte + var x10086 [1 << 17]byte + var x10087 [1 << 17]byte + var x10088 [1 << 17]byte + var x10089 [1 << 17]byte + var x10090 [1 << 17]byte + var x10091 [1 << 17]byte + var x10092 [1 << 17]byte + var x10093 [1 << 17]byte + var x10094 [1 << 17]byte + var x10095 [1 << 17]byte + var x10096 [1 << 17]byte + var x10097 [1 << 17]byte + var x10098 [1 << 17]byte + var x10099 [1 << 17]byte + var x10100 [1 << 17]byte + var x10101 [1 << 17]byte + var x10102 [1 << 17]byte + var x10103 [1 << 17]byte + var x10104 [1 << 17]byte + var x10105 [1 << 17]byte + var x10106 [1 << 17]byte + var x10107 [1 << 17]byte + var x10108 [1 << 17]byte + var x10109 [1 << 17]byte + var x10110 [1 << 17]byte + var x10111 [1 << 17]byte + var x10112 [1 << 17]byte + var x10113 [1 << 17]byte + var x10114 [1 << 17]byte + var x10115 [1 << 17]byte + var x10116 [1 << 17]byte + var x10117 [1 << 17]byte + var x10118 [1 << 17]byte + var x10119 [1 << 17]byte + var x10120 [1 << 17]byte + var x10121 [1 << 17]byte + var x10122 [1 << 17]byte + var x10123 [1 << 17]byte + var x10124 [1 << 17]byte + var x10125 [1 << 17]byte + var x10126 [1 << 17]byte + var x10127 [1 << 17]byte + var x10128 [1 << 17]byte + var x10129 [1 << 17]byte + var x10130 [1 << 17]byte + var x10131 [1 << 17]byte + var x10132 [1 << 17]byte + var x10133 [1 << 17]byte + var x10134 [1 << 17]byte + var x10135 [1 << 17]byte + var x10136 [1 << 17]byte + var x10137 [1 << 17]byte + var x10138 [1 << 17]byte + var x10139 [1 << 17]byte + var x10140 [1 << 17]byte + var x10141 [1 << 17]byte + var x10142 [1 << 17]byte + var x10143 [1 << 17]byte + var x10144 [1 << 17]byte + var x10145 [1 << 17]byte + var x10146 [1 << 17]byte + var x10147 [1 << 17]byte + var x10148 [1 << 17]byte + var x10149 [1 << 17]byte + var x10150 [1 << 17]byte + var x10151 [1 << 17]byte + var x10152 [1 << 17]byte + var x10153 [1 << 17]byte + var x10154 [1 << 17]byte + var x10155 [1 << 17]byte + var x10156 [1 << 17]byte + var x10157 [1 << 17]byte + var x10158 [1 << 17]byte + var x10159 [1 << 17]byte + var x10160 [1 << 17]byte + var x10161 [1 << 17]byte + var x10162 [1 << 17]byte + var x10163 [1 << 17]byte + var x10164 [1 << 17]byte + var x10165 [1 << 17]byte + var x10166 [1 << 17]byte + var x10167 [1 << 17]byte + var x10168 [1 << 17]byte + var x10169 [1 << 17]byte + var x10170 [1 << 17]byte + var x10171 [1 << 17]byte + var x10172 [1 << 17]byte + var x10173 [1 << 17]byte + var x10174 [1 << 17]byte + var x10175 [1 << 17]byte + var x10176 [1 << 17]byte + var x10177 [1 << 17]byte + var x10178 [1 << 17]byte + var x10179 [1 << 17]byte + var x10180 [1 << 17]byte + var x10181 [1 << 17]byte + var x10182 [1 << 17]byte + var x10183 [1 << 17]byte + var x10184 [1 << 17]byte + var x10185 [1 << 17]byte + var x10186 [1 << 17]byte + var x10187 [1 << 17]byte + var x10188 [1 << 17]byte + var x10189 [1 << 17]byte + var x10190 [1 << 17]byte + var x10191 [1 << 17]byte + var x10192 [1 << 17]byte + var x10193 [1 << 17]byte + var x10194 [1 << 17]byte + var x10195 [1 << 17]byte + var x10196 [1 << 17]byte + var x10197 [1 << 17]byte + var x10198 [1 << 17]byte + var x10199 [1 << 17]byte + var x10200 [1 << 17]byte + var x10201 [1 << 17]byte + var x10202 [1 << 17]byte + var x10203 [1 << 17]byte + var x10204 [1 << 17]byte + var x10205 [1 << 17]byte + var x10206 [1 << 17]byte + var x10207 [1 << 17]byte + var x10208 [1 << 17]byte + var x10209 [1 << 17]byte + var x10210 [1 << 17]byte + var x10211 [1 << 17]byte + var x10212 [1 << 17]byte + var x10213 [1 << 17]byte + var x10214 [1 << 17]byte + var x10215 [1 << 17]byte + var x10216 [1 << 17]byte + var x10217 [1 << 17]byte + var x10218 [1 << 17]byte + var x10219 [1 << 17]byte + var x10220 [1 << 17]byte + var x10221 [1 << 17]byte + var x10222 [1 << 17]byte + var x10223 [1 << 17]byte + var x10224 [1 << 17]byte + var x10225 [1 << 17]byte + var x10226 [1 << 17]byte + var x10227 [1 << 17]byte + var x10228 [1 << 17]byte + var x10229 [1 << 17]byte + var x10230 [1 << 17]byte + var x10231 [1 << 17]byte + var x10232 [1 << 17]byte + var x10233 [1 << 17]byte + var x10234 [1 << 17]byte + var x10235 [1 << 17]byte + var x10236 [1 << 17]byte + var x10237 [1 << 17]byte + var x10238 [1 << 17]byte + var x10239 [1 << 17]byte + var x10240 [1 << 17]byte + var x10241 [1 << 17]byte + var x10242 [1 << 17]byte + var x10243 [1 << 17]byte + var x10244 [1 << 17]byte + var x10245 [1 << 17]byte + var x10246 [1 << 17]byte + var x10247 [1 << 17]byte + var x10248 [1 << 17]byte + var x10249 [1 << 17]byte + var x10250 [1 << 17]byte + var x10251 [1 << 17]byte + var x10252 [1 << 17]byte + var x10253 [1 << 17]byte + var x10254 [1 << 17]byte + var x10255 [1 << 17]byte + var x10256 [1 << 17]byte + var x10257 [1 << 17]byte + var x10258 [1 << 17]byte + var x10259 [1 << 17]byte + var x10260 [1 << 17]byte + var x10261 [1 << 17]byte + var x10262 [1 << 17]byte + var x10263 [1 << 17]byte + var x10264 [1 << 17]byte + var x10265 [1 << 17]byte + var x10266 [1 << 17]byte + var x10267 [1 << 17]byte + var x10268 [1 << 17]byte + var x10269 [1 << 17]byte + var x10270 [1 << 17]byte + var x10271 [1 << 17]byte + var x10272 [1 << 17]byte + var x10273 [1 << 17]byte + var x10274 [1 << 17]byte + var x10275 [1 << 17]byte + var x10276 [1 << 17]byte + var x10277 [1 << 17]byte + var x10278 [1 << 17]byte + var x10279 [1 << 17]byte + var x10280 [1 << 17]byte + var x10281 [1 << 17]byte + var x10282 [1 << 17]byte + var x10283 [1 << 17]byte + var x10284 [1 << 17]byte + var x10285 [1 << 17]byte + var x10286 [1 << 17]byte + var x10287 [1 << 17]byte + var x10288 [1 << 17]byte + var x10289 [1 << 17]byte + var x10290 [1 << 17]byte + var x10291 [1 << 17]byte + var x10292 [1 << 17]byte + var x10293 [1 << 17]byte + var x10294 [1 << 17]byte + var x10295 [1 << 17]byte + var x10296 [1 << 17]byte + var x10297 [1 << 17]byte + var x10298 [1 << 17]byte + var x10299 [1 << 17]byte + var x10300 [1 << 17]byte + var x10301 [1 << 17]byte + var x10302 [1 << 17]byte + var x10303 [1 << 17]byte + var x10304 [1 << 17]byte + var x10305 [1 << 17]byte + var x10306 [1 << 17]byte + var x10307 [1 << 17]byte + var x10308 [1 << 17]byte + var x10309 [1 << 17]byte + var x10310 [1 << 17]byte + var x10311 [1 << 17]byte + var x10312 [1 << 17]byte + var x10313 [1 << 17]byte + var x10314 [1 << 17]byte + var x10315 [1 << 17]byte + var x10316 [1 << 17]byte + var x10317 [1 << 17]byte + var x10318 [1 << 17]byte + var x10319 [1 << 17]byte + var x10320 [1 << 17]byte + var x10321 [1 << 17]byte + var x10322 [1 << 17]byte + var x10323 [1 << 17]byte + var x10324 [1 << 17]byte + var x10325 [1 << 17]byte + var x10326 [1 << 17]byte + var x10327 [1 << 17]byte + var x10328 [1 << 17]byte + var x10329 [1 << 17]byte + var x10330 [1 << 17]byte + var x10331 [1 << 17]byte + var x10332 [1 << 17]byte + var x10333 [1 << 17]byte + var x10334 [1 << 17]byte + var x10335 [1 << 17]byte + var x10336 [1 << 17]byte + var x10337 [1 << 17]byte + var x10338 [1 << 17]byte + var x10339 [1 << 17]byte + var x10340 [1 << 17]byte + var x10341 [1 << 17]byte + var x10342 [1 << 17]byte + var x10343 [1 << 17]byte + var x10344 [1 << 17]byte + var x10345 [1 << 17]byte + var x10346 [1 << 17]byte + var x10347 [1 << 17]byte + var x10348 [1 << 17]byte + var x10349 [1 << 17]byte + var x10350 [1 << 17]byte + var x10351 [1 << 17]byte + var x10352 [1 << 17]byte + var x10353 [1 << 17]byte + var x10354 [1 << 17]byte + var x10355 [1 << 17]byte + var x10356 [1 << 17]byte + var x10357 [1 << 17]byte + var x10358 [1 << 17]byte + var x10359 [1 << 17]byte + var x10360 [1 << 17]byte + var x10361 [1 << 17]byte + var x10362 [1 << 17]byte + var x10363 [1 << 17]byte + var x10364 [1 << 17]byte + var x10365 [1 << 17]byte + var x10366 [1 << 17]byte + var x10367 [1 << 17]byte + var x10368 [1 << 17]byte + var x10369 [1 << 17]byte + var x10370 [1 << 17]byte + var x10371 [1 << 17]byte + var x10372 [1 << 17]byte + var x10373 [1 << 17]byte + var x10374 [1 << 17]byte + var x10375 [1 << 17]byte + var x10376 [1 << 17]byte + var x10377 [1 << 17]byte + var x10378 [1 << 17]byte + var x10379 [1 << 17]byte + var x10380 [1 << 17]byte + var x10381 [1 << 17]byte + var x10382 [1 << 17]byte + var x10383 [1 << 17]byte + var x10384 [1 << 17]byte + var x10385 [1 << 17]byte + var x10386 [1 << 17]byte + var x10387 [1 << 17]byte + var x10388 [1 << 17]byte + var x10389 [1 << 17]byte + var x10390 [1 << 17]byte + var x10391 [1 << 17]byte + var x10392 [1 << 17]byte + var x10393 [1 << 17]byte + var x10394 [1 << 17]byte + var x10395 [1 << 17]byte + var x10396 [1 << 17]byte + var x10397 [1 << 17]byte + var x10398 [1 << 17]byte + var x10399 [1 << 17]byte + var x10400 [1 << 17]byte + var x10401 [1 << 17]byte + var x10402 [1 << 17]byte + var x10403 [1 << 17]byte + var x10404 [1 << 17]byte + var x10405 [1 << 17]byte + var x10406 [1 << 17]byte + var x10407 [1 << 17]byte + var x10408 [1 << 17]byte + var x10409 [1 << 17]byte + var x10410 [1 << 17]byte + var x10411 [1 << 17]byte + var x10412 [1 << 17]byte + var x10413 [1 << 17]byte + var x10414 [1 << 17]byte + var x10415 [1 << 17]byte + var x10416 [1 << 17]byte + var x10417 [1 << 17]byte + var x10418 [1 << 17]byte + var x10419 [1 << 17]byte + var x10420 [1 << 17]byte + var x10421 [1 << 17]byte + var x10422 [1 << 17]byte + var x10423 [1 << 17]byte + var x10424 [1 << 17]byte + var x10425 [1 << 17]byte + var x10426 [1 << 17]byte + var x10427 [1 << 17]byte + var x10428 [1 << 17]byte + var x10429 [1 << 17]byte + var x10430 [1 << 17]byte + var x10431 [1 << 17]byte + var x10432 [1 << 17]byte + var x10433 [1 << 17]byte + var x10434 [1 << 17]byte + var x10435 [1 << 17]byte + var x10436 [1 << 17]byte + var x10437 [1 << 17]byte + var x10438 [1 << 17]byte + var x10439 [1 << 17]byte + var x10440 [1 << 17]byte + var x10441 [1 << 17]byte + var x10442 [1 << 17]byte + var x10443 [1 << 17]byte + var x10444 [1 << 17]byte + var x10445 [1 << 17]byte + var x10446 [1 << 17]byte + var x10447 [1 << 17]byte + var x10448 [1 << 17]byte + var x10449 [1 << 17]byte + var x10450 [1 << 17]byte + var x10451 [1 << 17]byte + var x10452 [1 << 17]byte + var x10453 [1 << 17]byte + var x10454 [1 << 17]byte + var x10455 [1 << 17]byte + var x10456 [1 << 17]byte + var x10457 [1 << 17]byte + var x10458 [1 << 17]byte + var x10459 [1 << 17]byte + var x10460 [1 << 17]byte + var x10461 [1 << 17]byte + var x10462 [1 << 17]byte + var x10463 [1 << 17]byte + var x10464 [1 << 17]byte + var x10465 [1 << 17]byte + var x10466 [1 << 17]byte + var x10467 [1 << 17]byte + var x10468 [1 << 17]byte + var x10469 [1 << 17]byte + var x10470 [1 << 17]byte + var x10471 [1 << 17]byte + var x10472 [1 << 17]byte + var x10473 [1 << 17]byte + var x10474 [1 << 17]byte + var x10475 [1 << 17]byte + var x10476 [1 << 17]byte + var x10477 [1 << 17]byte + var x10478 [1 << 17]byte + var x10479 [1 << 17]byte + var x10480 [1 << 17]byte + var x10481 [1 << 17]byte + var x10482 [1 << 17]byte + var x10483 [1 << 17]byte + var x10484 [1 << 17]byte + var x10485 [1 << 17]byte + var x10486 [1 << 17]byte + var x10487 [1 << 17]byte + var x10488 [1 << 17]byte + var x10489 [1 << 17]byte + var x10490 [1 << 17]byte + var x10491 [1 << 17]byte + var x10492 [1 << 17]byte + var x10493 [1 << 17]byte + var x10494 [1 << 17]byte + var x10495 [1 << 17]byte + var x10496 [1 << 17]byte + var x10497 [1 << 17]byte + var x10498 [1 << 17]byte + var x10499 [1 << 17]byte + var x10500 [1 << 17]byte + var x10501 [1 << 17]byte + var x10502 [1 << 17]byte + var x10503 [1 << 17]byte + var x10504 [1 << 17]byte + var x10505 [1 << 17]byte + var x10506 [1 << 17]byte + var x10507 [1 << 17]byte + var x10508 [1 << 17]byte + var x10509 [1 << 17]byte + var x10510 [1 << 17]byte + var x10511 [1 << 17]byte + var x10512 [1 << 17]byte + var x10513 [1 << 17]byte + var x10514 [1 << 17]byte + var x10515 [1 << 17]byte + var x10516 [1 << 17]byte + var x10517 [1 << 17]byte + var x10518 [1 << 17]byte + var x10519 [1 << 17]byte + var x10520 [1 << 17]byte + var x10521 [1 << 17]byte + var x10522 [1 << 17]byte + var x10523 [1 << 17]byte + var x10524 [1 << 17]byte + var x10525 [1 << 17]byte + var x10526 [1 << 17]byte + var x10527 [1 << 17]byte + var x10528 [1 << 17]byte + var x10529 [1 << 17]byte + var x10530 [1 << 17]byte + var x10531 [1 << 17]byte + var x10532 [1 << 17]byte + var x10533 [1 << 17]byte + var x10534 [1 << 17]byte + var x10535 [1 << 17]byte + var x10536 [1 << 17]byte + var x10537 [1 << 17]byte + var x10538 [1 << 17]byte + var x10539 [1 << 17]byte + var x10540 [1 << 17]byte + var x10541 [1 << 17]byte + var x10542 [1 << 17]byte + var x10543 [1 << 17]byte + var x10544 [1 << 17]byte + var x10545 [1 << 17]byte + var x10546 [1 << 17]byte + var x10547 [1 << 17]byte + var x10548 [1 << 17]byte + var x10549 [1 << 17]byte + var x10550 [1 << 17]byte + var x10551 [1 << 17]byte + var x10552 [1 << 17]byte + var x10553 [1 << 17]byte + var x10554 [1 << 17]byte + var x10555 [1 << 17]byte + var x10556 [1 << 17]byte + var x10557 [1 << 17]byte + var x10558 [1 << 17]byte + var x10559 [1 << 17]byte + var x10560 [1 << 17]byte + var x10561 [1 << 17]byte + var x10562 [1 << 17]byte + var x10563 [1 << 17]byte + var x10564 [1 << 17]byte + var x10565 [1 << 17]byte + var x10566 [1 << 17]byte + var x10567 [1 << 17]byte + var x10568 [1 << 17]byte + var x10569 [1 << 17]byte + var x10570 [1 << 17]byte + var x10571 [1 << 17]byte + var x10572 [1 << 17]byte + var x10573 [1 << 17]byte + var x10574 [1 << 17]byte + var x10575 [1 << 17]byte + var x10576 [1 << 17]byte + var x10577 [1 << 17]byte + var x10578 [1 << 17]byte + var x10579 [1 << 17]byte + var x10580 [1 << 17]byte + var x10581 [1 << 17]byte + var x10582 [1 << 17]byte + var x10583 [1 << 17]byte + var x10584 [1 << 17]byte + var x10585 [1 << 17]byte + var x10586 [1 << 17]byte + var x10587 [1 << 17]byte + var x10588 [1 << 17]byte + var x10589 [1 << 17]byte + var x10590 [1 << 17]byte + var x10591 [1 << 17]byte + var x10592 [1 << 17]byte + var x10593 [1 << 17]byte + var x10594 [1 << 17]byte + var x10595 [1 << 17]byte + var x10596 [1 << 17]byte + var x10597 [1 << 17]byte + var x10598 [1 << 17]byte + var x10599 [1 << 17]byte + var x10600 [1 << 17]byte + var x10601 [1 << 17]byte + var x10602 [1 << 17]byte + var x10603 [1 << 17]byte + var x10604 [1 << 17]byte + var x10605 [1 << 17]byte + var x10606 [1 << 17]byte + var x10607 [1 << 17]byte + var x10608 [1 << 17]byte + var x10609 [1 << 17]byte + var x10610 [1 << 17]byte + var x10611 [1 << 17]byte + var x10612 [1 << 17]byte + var x10613 [1 << 17]byte + var x10614 [1 << 17]byte + var x10615 [1 << 17]byte + var x10616 [1 << 17]byte + var x10617 [1 << 17]byte + var x10618 [1 << 17]byte + var x10619 [1 << 17]byte + var x10620 [1 << 17]byte + var x10621 [1 << 17]byte + var x10622 [1 << 17]byte + var x10623 [1 << 17]byte + var x10624 [1 << 17]byte + var x10625 [1 << 17]byte + var x10626 [1 << 17]byte + var x10627 [1 << 17]byte + var x10628 [1 << 17]byte + var x10629 [1 << 17]byte + var x10630 [1 << 17]byte + var x10631 [1 << 17]byte + var x10632 [1 << 17]byte + var x10633 [1 << 17]byte + var x10634 [1 << 17]byte + var x10635 [1 << 17]byte + var x10636 [1 << 17]byte + var x10637 [1 << 17]byte + var x10638 [1 << 17]byte + var x10639 [1 << 17]byte + var x10640 [1 << 17]byte + var x10641 [1 << 17]byte + var x10642 [1 << 17]byte + var x10643 [1 << 17]byte + var x10644 [1 << 17]byte + var x10645 [1 << 17]byte + var x10646 [1 << 17]byte + var x10647 [1 << 17]byte + var x10648 [1 << 17]byte + var x10649 [1 << 17]byte + var x10650 [1 << 17]byte + var x10651 [1 << 17]byte + var x10652 [1 << 17]byte + var x10653 [1 << 17]byte + var x10654 [1 << 17]byte + var x10655 [1 << 17]byte + var x10656 [1 << 17]byte + var x10657 [1 << 17]byte + var x10658 [1 << 17]byte + var x10659 [1 << 17]byte + var x10660 [1 << 17]byte + var x10661 [1 << 17]byte + var x10662 [1 << 17]byte + var x10663 [1 << 17]byte + var x10664 [1 << 17]byte + var x10665 [1 << 17]byte + var x10666 [1 << 17]byte + var x10667 [1 << 17]byte + var x10668 [1 << 17]byte + var x10669 [1 << 17]byte + var x10670 [1 << 17]byte + var x10671 [1 << 17]byte + var x10672 [1 << 17]byte + var x10673 [1 << 17]byte + var x10674 [1 << 17]byte + var x10675 [1 << 17]byte + var x10676 [1 << 17]byte + var x10677 [1 << 17]byte + var x10678 [1 << 17]byte + var x10679 [1 << 17]byte + var x10680 [1 << 17]byte + var x10681 [1 << 17]byte + var x10682 [1 << 17]byte + var x10683 [1 << 17]byte + var x10684 [1 << 17]byte + var x10685 [1 << 17]byte + var x10686 [1 << 17]byte + var x10687 [1 << 17]byte + var x10688 [1 << 17]byte + var x10689 [1 << 17]byte + var x10690 [1 << 17]byte + var x10691 [1 << 17]byte + var x10692 [1 << 17]byte + var x10693 [1 << 17]byte + var x10694 [1 << 17]byte + var x10695 [1 << 17]byte + var x10696 [1 << 17]byte + var x10697 [1 << 17]byte + var x10698 [1 << 17]byte + var x10699 [1 << 17]byte + var x10700 [1 << 17]byte + var x10701 [1 << 17]byte + var x10702 [1 << 17]byte + var x10703 [1 << 17]byte + var x10704 [1 << 17]byte + var x10705 [1 << 17]byte + var x10706 [1 << 17]byte + var x10707 [1 << 17]byte + var x10708 [1 << 17]byte + var x10709 [1 << 17]byte + var x10710 [1 << 17]byte + var x10711 [1 << 17]byte + var x10712 [1 << 17]byte + var x10713 [1 << 17]byte + var x10714 [1 << 17]byte + var x10715 [1 << 17]byte + var x10716 [1 << 17]byte + var x10717 [1 << 17]byte + var x10718 [1 << 17]byte + var x10719 [1 << 17]byte + var x10720 [1 << 17]byte + var x10721 [1 << 17]byte + var x10722 [1 << 17]byte + var x10723 [1 << 17]byte + var x10724 [1 << 17]byte + var x10725 [1 << 17]byte + var x10726 [1 << 17]byte + var x10727 [1 << 17]byte + var x10728 [1 << 17]byte + var x10729 [1 << 17]byte + var x10730 [1 << 17]byte + var x10731 [1 << 17]byte + var x10732 [1 << 17]byte + var x10733 [1 << 17]byte + var x10734 [1 << 17]byte + var x10735 [1 << 17]byte + var x10736 [1 << 17]byte + var x10737 [1 << 17]byte + var x10738 [1 << 17]byte + var x10739 [1 << 17]byte + var x10740 [1 << 17]byte + var x10741 [1 << 17]byte + var x10742 [1 << 17]byte + var x10743 [1 << 17]byte + var x10744 [1 << 17]byte + var x10745 [1 << 17]byte + var x10746 [1 << 17]byte + var x10747 [1 << 17]byte + var x10748 [1 << 17]byte + var x10749 [1 << 17]byte + var x10750 [1 << 17]byte + var x10751 [1 << 17]byte + var x10752 [1 << 17]byte + var x10753 [1 << 17]byte + var x10754 [1 << 17]byte + var x10755 [1 << 17]byte + var x10756 [1 << 17]byte + var x10757 [1 << 17]byte + var x10758 [1 << 17]byte + var x10759 [1 << 17]byte + var x10760 [1 << 17]byte + var x10761 [1 << 17]byte + var x10762 [1 << 17]byte + var x10763 [1 << 17]byte + var x10764 [1 << 17]byte + var x10765 [1 << 17]byte + var x10766 [1 << 17]byte + var x10767 [1 << 17]byte + var x10768 [1 << 17]byte + var x10769 [1 << 17]byte + var x10770 [1 << 17]byte + var x10771 [1 << 17]byte + var x10772 [1 << 17]byte + var x10773 [1 << 17]byte + var x10774 [1 << 17]byte + var x10775 [1 << 17]byte + var x10776 [1 << 17]byte + var x10777 [1 << 17]byte + var x10778 [1 << 17]byte + var x10779 [1 << 17]byte + var x10780 [1 << 17]byte + var x10781 [1 << 17]byte + var x10782 [1 << 17]byte + var x10783 [1 << 17]byte + var x10784 [1 << 17]byte + var x10785 [1 << 17]byte + var x10786 [1 << 17]byte + var x10787 [1 << 17]byte + var x10788 [1 << 17]byte + var x10789 [1 << 17]byte + var x10790 [1 << 17]byte + var x10791 [1 << 17]byte + var x10792 [1 << 17]byte + var x10793 [1 << 17]byte + var x10794 [1 << 17]byte + var x10795 [1 << 17]byte + var x10796 [1 << 17]byte + var x10797 [1 << 17]byte + var x10798 [1 << 17]byte + var x10799 [1 << 17]byte + var x10800 [1 << 17]byte + var x10801 [1 << 17]byte + var x10802 [1 << 17]byte + var x10803 [1 << 17]byte + var x10804 [1 << 17]byte + var x10805 [1 << 17]byte + var x10806 [1 << 17]byte + var x10807 [1 << 17]byte + var x10808 [1 << 17]byte + var x10809 [1 << 17]byte + var x10810 [1 << 17]byte + var x10811 [1 << 17]byte + var x10812 [1 << 17]byte + var x10813 [1 << 17]byte + var x10814 [1 << 17]byte + var x10815 [1 << 17]byte + var x10816 [1 << 17]byte + var x10817 [1 << 17]byte + var x10818 [1 << 17]byte + var x10819 [1 << 17]byte + var x10820 [1 << 17]byte + var x10821 [1 << 17]byte + var x10822 [1 << 17]byte + var x10823 [1 << 17]byte + var x10824 [1 << 17]byte + var x10825 [1 << 17]byte + var x10826 [1 << 17]byte + var x10827 [1 << 17]byte + var x10828 [1 << 17]byte + var x10829 [1 << 17]byte + var x10830 [1 << 17]byte + var x10831 [1 << 17]byte + var x10832 [1 << 17]byte + var x10833 [1 << 17]byte + var x10834 [1 << 17]byte + var x10835 [1 << 17]byte + var x10836 [1 << 17]byte + var x10837 [1 << 17]byte + var x10838 [1 << 17]byte + var x10839 [1 << 17]byte + var x10840 [1 << 17]byte + var x10841 [1 << 17]byte + var x10842 [1 << 17]byte + var x10843 [1 << 17]byte + var x10844 [1 << 17]byte + var x10845 [1 << 17]byte + var x10846 [1 << 17]byte + var x10847 [1 << 17]byte + var x10848 [1 << 17]byte + var x10849 [1 << 17]byte + var x10850 [1 << 17]byte + var x10851 [1 << 17]byte + var x10852 [1 << 17]byte + var x10853 [1 << 17]byte + var x10854 [1 << 17]byte + var x10855 [1 << 17]byte + var x10856 [1 << 17]byte + var x10857 [1 << 17]byte + var x10858 [1 << 17]byte + var x10859 [1 << 17]byte + var x10860 [1 << 17]byte + var x10861 [1 << 17]byte + var x10862 [1 << 17]byte + var x10863 [1 << 17]byte + var x10864 [1 << 17]byte + var x10865 [1 << 17]byte + var x10866 [1 << 17]byte + var x10867 [1 << 17]byte + var x10868 [1 << 17]byte + var x10869 [1 << 17]byte + var x10870 [1 << 17]byte + var x10871 [1 << 17]byte + var x10872 [1 << 17]byte + var x10873 [1 << 17]byte + var x10874 [1 << 17]byte + var x10875 [1 << 17]byte + var x10876 [1 << 17]byte + var x10877 [1 << 17]byte + var x10878 [1 << 17]byte + var x10879 [1 << 17]byte + var x10880 [1 << 17]byte + var x10881 [1 << 17]byte + var x10882 [1 << 17]byte + var x10883 [1 << 17]byte + var x10884 [1 << 17]byte + var x10885 [1 << 17]byte + var x10886 [1 << 17]byte + var x10887 [1 << 17]byte + var x10888 [1 << 17]byte + var x10889 [1 << 17]byte + var x10890 [1 << 17]byte + var x10891 [1 << 17]byte + var x10892 [1 << 17]byte + var x10893 [1 << 17]byte + var x10894 [1 << 17]byte + var x10895 [1 << 17]byte + var x10896 [1 << 17]byte + var x10897 [1 << 17]byte + var x10898 [1 << 17]byte + var x10899 [1 << 17]byte + var x10900 [1 << 17]byte + var x10901 [1 << 17]byte + var x10902 [1 << 17]byte + var x10903 [1 << 17]byte + var x10904 [1 << 17]byte + var x10905 [1 << 17]byte + var x10906 [1 << 17]byte + var x10907 [1 << 17]byte + var x10908 [1 << 17]byte + var x10909 [1 << 17]byte + var x10910 [1 << 17]byte + var x10911 [1 << 17]byte + var x10912 [1 << 17]byte + var x10913 [1 << 17]byte + var x10914 [1 << 17]byte + var x10915 [1 << 17]byte + var x10916 [1 << 17]byte + var x10917 [1 << 17]byte + var x10918 [1 << 17]byte + var x10919 [1 << 17]byte + var x10920 [1 << 17]byte + var x10921 [1 << 17]byte + var x10922 [1 << 17]byte + var x10923 [1 << 17]byte + var x10924 [1 << 17]byte + var x10925 [1 << 17]byte + var x10926 [1 << 17]byte + var x10927 [1 << 17]byte + var x10928 [1 << 17]byte + var x10929 [1 << 17]byte + var x10930 [1 << 17]byte + var x10931 [1 << 17]byte + var x10932 [1 << 17]byte + var x10933 [1 << 17]byte + var x10934 [1 << 17]byte + var x10935 [1 << 17]byte + var x10936 [1 << 17]byte + var x10937 [1 << 17]byte + var x10938 [1 << 17]byte + var x10939 [1 << 17]byte + var x10940 [1 << 17]byte + var x10941 [1 << 17]byte + var x10942 [1 << 17]byte + var x10943 [1 << 17]byte + var x10944 [1 << 17]byte + var x10945 [1 << 17]byte + var x10946 [1 << 17]byte + var x10947 [1 << 17]byte + var x10948 [1 << 17]byte + var x10949 [1 << 17]byte + var x10950 [1 << 17]byte + var x10951 [1 << 17]byte + var x10952 [1 << 17]byte + var x10953 [1 << 17]byte + var x10954 [1 << 17]byte + var x10955 [1 << 17]byte + var x10956 [1 << 17]byte + var x10957 [1 << 17]byte + var x10958 [1 << 17]byte + var x10959 [1 << 17]byte + var x10960 [1 << 17]byte + var x10961 [1 << 17]byte + var x10962 [1 << 17]byte + var x10963 [1 << 17]byte + var x10964 [1 << 17]byte + var x10965 [1 << 17]byte + var x10966 [1 << 17]byte + var x10967 [1 << 17]byte + var x10968 [1 << 17]byte + var x10969 [1 << 17]byte + var x10970 [1 << 17]byte + var x10971 [1 << 17]byte + var x10972 [1 << 17]byte + var x10973 [1 << 17]byte + var x10974 [1 << 17]byte + var x10975 [1 << 17]byte + var x10976 [1 << 17]byte + var x10977 [1 << 17]byte + var x10978 [1 << 17]byte + var x10979 [1 << 17]byte + var x10980 [1 << 17]byte + var x10981 [1 << 17]byte + var x10982 [1 << 17]byte + var x10983 [1 << 17]byte + var x10984 [1 << 17]byte + var x10985 [1 << 17]byte + var x10986 [1 << 17]byte + var x10987 [1 << 17]byte + var x10988 [1 << 17]byte + var x10989 [1 << 17]byte + var x10990 [1 << 17]byte + var x10991 [1 << 17]byte + var x10992 [1 << 17]byte + var x10993 [1 << 17]byte + var x10994 [1 << 17]byte + var x10995 [1 << 17]byte + var x10996 [1 << 17]byte + var x10997 [1 << 17]byte + var x10998 [1 << 17]byte + var x10999 [1 << 17]byte + var x11000 [1 << 17]byte + var x11001 [1 << 17]byte + var x11002 [1 << 17]byte + var x11003 [1 << 17]byte + var x11004 [1 << 17]byte + var x11005 [1 << 17]byte + var x11006 [1 << 17]byte + var x11007 [1 << 17]byte + var x11008 [1 << 17]byte + var x11009 [1 << 17]byte + var x11010 [1 << 17]byte + var x11011 [1 << 17]byte + var x11012 [1 << 17]byte + var x11013 [1 << 17]byte + var x11014 [1 << 17]byte + var x11015 [1 << 17]byte + var x11016 [1 << 17]byte + var x11017 [1 << 17]byte + var x11018 [1 << 17]byte + var x11019 [1 << 17]byte + var x11020 [1 << 17]byte + var x11021 [1 << 17]byte + var x11022 [1 << 17]byte + var x11023 [1 << 17]byte + var x11024 [1 << 17]byte + var x11025 [1 << 17]byte + var x11026 [1 << 17]byte + var x11027 [1 << 17]byte + var x11028 [1 << 17]byte + var x11029 [1 << 17]byte + var x11030 [1 << 17]byte + var x11031 [1 << 17]byte + var x11032 [1 << 17]byte + var x11033 [1 << 17]byte + var x11034 [1 << 17]byte + var x11035 [1 << 17]byte + var x11036 [1 << 17]byte + var x11037 [1 << 17]byte + var x11038 [1 << 17]byte + var x11039 [1 << 17]byte + var x11040 [1 << 17]byte + var x11041 [1 << 17]byte + var x11042 [1 << 17]byte + var x11043 [1 << 17]byte + var x11044 [1 << 17]byte + var x11045 [1 << 17]byte + var x11046 [1 << 17]byte + var x11047 [1 << 17]byte + var x11048 [1 << 17]byte + var x11049 [1 << 17]byte + var x11050 [1 << 17]byte + var x11051 [1 << 17]byte + var x11052 [1 << 17]byte + var x11053 [1 << 17]byte + var x11054 [1 << 17]byte + var x11055 [1 << 17]byte + var x11056 [1 << 17]byte + var x11057 [1 << 17]byte + var x11058 [1 << 17]byte + var x11059 [1 << 17]byte + var x11060 [1 << 17]byte + var x11061 [1 << 17]byte + var x11062 [1 << 17]byte + var x11063 [1 << 17]byte + var x11064 [1 << 17]byte + var x11065 [1 << 17]byte + var x11066 [1 << 17]byte + var x11067 [1 << 17]byte + var x11068 [1 << 17]byte + var x11069 [1 << 17]byte + var x11070 [1 << 17]byte + var x11071 [1 << 17]byte + var x11072 [1 << 17]byte + var x11073 [1 << 17]byte + var x11074 [1 << 17]byte + var x11075 [1 << 17]byte + var x11076 [1 << 17]byte + var x11077 [1 << 17]byte + var x11078 [1 << 17]byte + var x11079 [1 << 17]byte + var x11080 [1 << 17]byte + var x11081 [1 << 17]byte + var x11082 [1 << 17]byte + var x11083 [1 << 17]byte + var x11084 [1 << 17]byte + var x11085 [1 << 17]byte + var x11086 [1 << 17]byte + var x11087 [1 << 17]byte + var x11088 [1 << 17]byte + var x11089 [1 << 17]byte + var x11090 [1 << 17]byte + var x11091 [1 << 17]byte + var x11092 [1 << 17]byte + var x11093 [1 << 17]byte + var x11094 [1 << 17]byte + var x11095 [1 << 17]byte + var x11096 [1 << 17]byte + var x11097 [1 << 17]byte + var x11098 [1 << 17]byte + var x11099 [1 << 17]byte + var x11100 [1 << 17]byte + var x11101 [1 << 17]byte + var x11102 [1 << 17]byte + var x11103 [1 << 17]byte + var x11104 [1 << 17]byte + var x11105 [1 << 17]byte + var x11106 [1 << 17]byte + var x11107 [1 << 17]byte + var x11108 [1 << 17]byte + var x11109 [1 << 17]byte + var x11110 [1 << 17]byte + var x11111 [1 << 17]byte + var x11112 [1 << 17]byte + var x11113 [1 << 17]byte + var x11114 [1 << 17]byte + var x11115 [1 << 17]byte + var x11116 [1 << 17]byte + var x11117 [1 << 17]byte + var x11118 [1 << 17]byte + var x11119 [1 << 17]byte + var x11120 [1 << 17]byte + var x11121 [1 << 17]byte + var x11122 [1 << 17]byte + var x11123 [1 << 17]byte + var x11124 [1 << 17]byte + var x11125 [1 << 17]byte + var x11126 [1 << 17]byte + var x11127 [1 << 17]byte + var x11128 [1 << 17]byte + var x11129 [1 << 17]byte + var x11130 [1 << 17]byte + var x11131 [1 << 17]byte + var x11132 [1 << 17]byte + var x11133 [1 << 17]byte + var x11134 [1 << 17]byte + var x11135 [1 << 17]byte + var x11136 [1 << 17]byte + var x11137 [1 << 17]byte + var x11138 [1 << 17]byte + var x11139 [1 << 17]byte + var x11140 [1 << 17]byte + var x11141 [1 << 17]byte + var x11142 [1 << 17]byte + var x11143 [1 << 17]byte + var x11144 [1 << 17]byte + var x11145 [1 << 17]byte + var x11146 [1 << 17]byte + var x11147 [1 << 17]byte + var x11148 [1 << 17]byte + var x11149 [1 << 17]byte + var x11150 [1 << 17]byte + var x11151 [1 << 17]byte + var x11152 [1 << 17]byte + var x11153 [1 << 17]byte + var x11154 [1 << 17]byte + var x11155 [1 << 17]byte + var x11156 [1 << 17]byte + var x11157 [1 << 17]byte + var x11158 [1 << 17]byte + var x11159 [1 << 17]byte + var x11160 [1 << 17]byte + var x11161 [1 << 17]byte + var x11162 [1 << 17]byte + var x11163 [1 << 17]byte + var x11164 [1 << 17]byte + var x11165 [1 << 17]byte + var x11166 [1 << 17]byte + var x11167 [1 << 17]byte + var x11168 [1 << 17]byte + var x11169 [1 << 17]byte + var x11170 [1 << 17]byte + var x11171 [1 << 17]byte + var x11172 [1 << 17]byte + var x11173 [1 << 17]byte + var x11174 [1 << 17]byte + var x11175 [1 << 17]byte + var x11176 [1 << 17]byte + var x11177 [1 << 17]byte + var x11178 [1 << 17]byte + var x11179 [1 << 17]byte + var x11180 [1 << 17]byte + var x11181 [1 << 17]byte + var x11182 [1 << 17]byte + var x11183 [1 << 17]byte + var x11184 [1 << 17]byte + var x11185 [1 << 17]byte + var x11186 [1 << 17]byte + var x11187 [1 << 17]byte + var x11188 [1 << 17]byte + var x11189 [1 << 17]byte + var x11190 [1 << 17]byte + var x11191 [1 << 17]byte + var x11192 [1 << 17]byte + var x11193 [1 << 17]byte + var x11194 [1 << 17]byte + var x11195 [1 << 17]byte + var x11196 [1 << 17]byte + var x11197 [1 << 17]byte + var x11198 [1 << 17]byte + var x11199 [1 << 17]byte + var x11200 [1 << 17]byte + var x11201 [1 << 17]byte + var x11202 [1 << 17]byte + var x11203 [1 << 17]byte + var x11204 [1 << 17]byte + var x11205 [1 << 17]byte + var x11206 [1 << 17]byte + var x11207 [1 << 17]byte + var x11208 [1 << 17]byte + var x11209 [1 << 17]byte + var x11210 [1 << 17]byte + var x11211 [1 << 17]byte + var x11212 [1 << 17]byte + var x11213 [1 << 17]byte + var x11214 [1 << 17]byte + var x11215 [1 << 17]byte + var x11216 [1 << 17]byte + var x11217 [1 << 17]byte + var x11218 [1 << 17]byte + var x11219 [1 << 17]byte + var x11220 [1 << 17]byte + var x11221 [1 << 17]byte + var x11222 [1 << 17]byte + var x11223 [1 << 17]byte + var x11224 [1 << 17]byte + var x11225 [1 << 17]byte + var x11226 [1 << 17]byte + var x11227 [1 << 17]byte + var x11228 [1 << 17]byte + var x11229 [1 << 17]byte + var x11230 [1 << 17]byte + var x11231 [1 << 17]byte + var x11232 [1 << 17]byte + var x11233 [1 << 17]byte + var x11234 [1 << 17]byte + var x11235 [1 << 17]byte + var x11236 [1 << 17]byte + var x11237 [1 << 17]byte + var x11238 [1 << 17]byte + var x11239 [1 << 17]byte + var x11240 [1 << 17]byte + var x11241 [1 << 17]byte + var x11242 [1 << 17]byte + var x11243 [1 << 17]byte + var x11244 [1 << 17]byte + var x11245 [1 << 17]byte + var x11246 [1 << 17]byte + var x11247 [1 << 17]byte + var x11248 [1 << 17]byte + var x11249 [1 << 17]byte + var x11250 [1 << 17]byte + var x11251 [1 << 17]byte + var x11252 [1 << 17]byte + var x11253 [1 << 17]byte + var x11254 [1 << 17]byte + var x11255 [1 << 17]byte + var x11256 [1 << 17]byte + var x11257 [1 << 17]byte + var x11258 [1 << 17]byte + var x11259 [1 << 17]byte + var x11260 [1 << 17]byte + var x11261 [1 << 17]byte + var x11262 [1 << 17]byte + var x11263 [1 << 17]byte + var x11264 [1 << 17]byte + var x11265 [1 << 17]byte + var x11266 [1 << 17]byte + var x11267 [1 << 17]byte + var x11268 [1 << 17]byte + var x11269 [1 << 17]byte + var x11270 [1 << 17]byte + var x11271 [1 << 17]byte + var x11272 [1 << 17]byte + var x11273 [1 << 17]byte + var x11274 [1 << 17]byte + var x11275 [1 << 17]byte + var x11276 [1 << 17]byte + var x11277 [1 << 17]byte + var x11278 [1 << 17]byte + var x11279 [1 << 17]byte + var x11280 [1 << 17]byte + var x11281 [1 << 17]byte + var x11282 [1 << 17]byte + var x11283 [1 << 17]byte + var x11284 [1 << 17]byte + var x11285 [1 << 17]byte + var x11286 [1 << 17]byte + var x11287 [1 << 17]byte + var x11288 [1 << 17]byte + var x11289 [1 << 17]byte + var x11290 [1 << 17]byte + var x11291 [1 << 17]byte + var x11292 [1 << 17]byte + var x11293 [1 << 17]byte + var x11294 [1 << 17]byte + var x11295 [1 << 17]byte + var x11296 [1 << 17]byte + var x11297 [1 << 17]byte + var x11298 [1 << 17]byte + var x11299 [1 << 17]byte + var x11300 [1 << 17]byte + var x11301 [1 << 17]byte + var x11302 [1 << 17]byte + var x11303 [1 << 17]byte + var x11304 [1 << 17]byte + var x11305 [1 << 17]byte + var x11306 [1 << 17]byte + var x11307 [1 << 17]byte + var x11308 [1 << 17]byte + var x11309 [1 << 17]byte + var x11310 [1 << 17]byte + var x11311 [1 << 17]byte + var x11312 [1 << 17]byte + var x11313 [1 << 17]byte + var x11314 [1 << 17]byte + var x11315 [1 << 17]byte + var x11316 [1 << 17]byte + var x11317 [1 << 17]byte + var x11318 [1 << 17]byte + var x11319 [1 << 17]byte + var x11320 [1 << 17]byte + var x11321 [1 << 17]byte + var x11322 [1 << 17]byte + var x11323 [1 << 17]byte + var x11324 [1 << 17]byte + var x11325 [1 << 17]byte + var x11326 [1 << 17]byte + var x11327 [1 << 17]byte + var x11328 [1 << 17]byte + var x11329 [1 << 17]byte + var x11330 [1 << 17]byte + var x11331 [1 << 17]byte + var x11332 [1 << 17]byte + var x11333 [1 << 17]byte + var x11334 [1 << 17]byte + var x11335 [1 << 17]byte + var x11336 [1 << 17]byte + var x11337 [1 << 17]byte + var x11338 [1 << 17]byte + var x11339 [1 << 17]byte + var x11340 [1 << 17]byte + var x11341 [1 << 17]byte + var x11342 [1 << 17]byte + var x11343 [1 << 17]byte + var x11344 [1 << 17]byte + var x11345 [1 << 17]byte + var x11346 [1 << 17]byte + var x11347 [1 << 17]byte + var x11348 [1 << 17]byte + var x11349 [1 << 17]byte + var x11350 [1 << 17]byte + var x11351 [1 << 17]byte + var x11352 [1 << 17]byte + var x11353 [1 << 17]byte + var x11354 [1 << 17]byte + var x11355 [1 << 17]byte + var x11356 [1 << 17]byte + var x11357 [1 << 17]byte + var x11358 [1 << 17]byte + var x11359 [1 << 17]byte + var x11360 [1 << 17]byte + var x11361 [1 << 17]byte + var x11362 [1 << 17]byte + var x11363 [1 << 17]byte + var x11364 [1 << 17]byte + var x11365 [1 << 17]byte + var x11366 [1 << 17]byte + var x11367 [1 << 17]byte + var x11368 [1 << 17]byte + var x11369 [1 << 17]byte + var x11370 [1 << 17]byte + var x11371 [1 << 17]byte + var x11372 [1 << 17]byte + var x11373 [1 << 17]byte + var x11374 [1 << 17]byte + var x11375 [1 << 17]byte + var x11376 [1 << 17]byte + var x11377 [1 << 17]byte + var x11378 [1 << 17]byte + var x11379 [1 << 17]byte + var x11380 [1 << 17]byte + var x11381 [1 << 17]byte + var x11382 [1 << 17]byte + var x11383 [1 << 17]byte + var x11384 [1 << 17]byte + var x11385 [1 << 17]byte + var x11386 [1 << 17]byte + var x11387 [1 << 17]byte + var x11388 [1 << 17]byte + var x11389 [1 << 17]byte + var x11390 [1 << 17]byte + var x11391 [1 << 17]byte + var x11392 [1 << 17]byte + var x11393 [1 << 17]byte + var x11394 [1 << 17]byte + var x11395 [1 << 17]byte + var x11396 [1 << 17]byte + var x11397 [1 << 17]byte + var x11398 [1 << 17]byte + var x11399 [1 << 17]byte + var x11400 [1 << 17]byte + var x11401 [1 << 17]byte + var x11402 [1 << 17]byte + var x11403 [1 << 17]byte + var x11404 [1 << 17]byte + var x11405 [1 << 17]byte + var x11406 [1 << 17]byte + var x11407 [1 << 17]byte + var x11408 [1 << 17]byte + var x11409 [1 << 17]byte + var x11410 [1 << 17]byte + var x11411 [1 << 17]byte + var x11412 [1 << 17]byte + var x11413 [1 << 17]byte + var x11414 [1 << 17]byte + var x11415 [1 << 17]byte + var x11416 [1 << 17]byte + var x11417 [1 << 17]byte + var x11418 [1 << 17]byte + var x11419 [1 << 17]byte + var x11420 [1 << 17]byte + var x11421 [1 << 17]byte + var x11422 [1 << 17]byte + var x11423 [1 << 17]byte + var x11424 [1 << 17]byte + var x11425 [1 << 17]byte + var x11426 [1 << 17]byte + var x11427 [1 << 17]byte + var x11428 [1 << 17]byte + var x11429 [1 << 17]byte + var x11430 [1 << 17]byte + var x11431 [1 << 17]byte + var x11432 [1 << 17]byte + var x11433 [1 << 17]byte + var x11434 [1 << 17]byte + var x11435 [1 << 17]byte + var x11436 [1 << 17]byte + var x11437 [1 << 17]byte + var x11438 [1 << 17]byte + var x11439 [1 << 17]byte + var x11440 [1 << 17]byte + var x11441 [1 << 17]byte + var x11442 [1 << 17]byte + var x11443 [1 << 17]byte + var x11444 [1 << 17]byte + var x11445 [1 << 17]byte + var x11446 [1 << 17]byte + var x11447 [1 << 17]byte + var x11448 [1 << 17]byte + var x11449 [1 << 17]byte + var x11450 [1 << 17]byte + var x11451 [1 << 17]byte + var x11452 [1 << 17]byte + var x11453 [1 << 17]byte + var x11454 [1 << 17]byte + var x11455 [1 << 17]byte + var x11456 [1 << 17]byte + var x11457 [1 << 17]byte + var x11458 [1 << 17]byte + var x11459 [1 << 17]byte + var x11460 [1 << 17]byte + var x11461 [1 << 17]byte + var x11462 [1 << 17]byte + var x11463 [1 << 17]byte + var x11464 [1 << 17]byte + var x11465 [1 << 17]byte + var x11466 [1 << 17]byte + var x11467 [1 << 17]byte + var x11468 [1 << 17]byte + var x11469 [1 << 17]byte + var x11470 [1 << 17]byte + var x11471 [1 << 17]byte + var x11472 [1 << 17]byte + var x11473 [1 << 17]byte + var x11474 [1 << 17]byte + var x11475 [1 << 17]byte + var x11476 [1 << 17]byte + var x11477 [1 << 17]byte + var x11478 [1 << 17]byte + var x11479 [1 << 17]byte + var x11480 [1 << 17]byte + var x11481 [1 << 17]byte + var x11482 [1 << 17]byte + var x11483 [1 << 17]byte + var x11484 [1 << 17]byte + var x11485 [1 << 17]byte + var x11486 [1 << 17]byte + var x11487 [1 << 17]byte + var x11488 [1 << 17]byte + var x11489 [1 << 17]byte + var x11490 [1 << 17]byte + var x11491 [1 << 17]byte + var x11492 [1 << 17]byte + var x11493 [1 << 17]byte + var x11494 [1 << 17]byte + var x11495 [1 << 17]byte + var x11496 [1 << 17]byte + var x11497 [1 << 17]byte + var x11498 [1 << 17]byte + var x11499 [1 << 17]byte + var x11500 [1 << 17]byte + var x11501 [1 << 17]byte + var x11502 [1 << 17]byte + var x11503 [1 << 17]byte + var x11504 [1 << 17]byte + var x11505 [1 << 17]byte + var x11506 [1 << 17]byte + var x11507 [1 << 17]byte + var x11508 [1 << 17]byte + var x11509 [1 << 17]byte + var x11510 [1 << 17]byte + var x11511 [1 << 17]byte + var x11512 [1 << 17]byte + var x11513 [1 << 17]byte + var x11514 [1 << 17]byte + var x11515 [1 << 17]byte + var x11516 [1 << 17]byte + var x11517 [1 << 17]byte + var x11518 [1 << 17]byte + var x11519 [1 << 17]byte + var x11520 [1 << 17]byte + var x11521 [1 << 17]byte + var x11522 [1 << 17]byte + var x11523 [1 << 17]byte + var x11524 [1 << 17]byte + var x11525 [1 << 17]byte + var x11526 [1 << 17]byte + var x11527 [1 << 17]byte + var x11528 [1 << 17]byte + var x11529 [1 << 17]byte + var x11530 [1 << 17]byte + var x11531 [1 << 17]byte + var x11532 [1 << 17]byte + var x11533 [1 << 17]byte + var x11534 [1 << 17]byte + var x11535 [1 << 17]byte + var x11536 [1 << 17]byte + var x11537 [1 << 17]byte + var x11538 [1 << 17]byte + var x11539 [1 << 17]byte + var x11540 [1 << 17]byte + var x11541 [1 << 17]byte + var x11542 [1 << 17]byte + var x11543 [1 << 17]byte + var x11544 [1 << 17]byte + var x11545 [1 << 17]byte + var x11546 [1 << 17]byte + var x11547 [1 << 17]byte + var x11548 [1 << 17]byte + var x11549 [1 << 17]byte + var x11550 [1 << 17]byte + var x11551 [1 << 17]byte + var x11552 [1 << 17]byte + var x11553 [1 << 17]byte + var x11554 [1 << 17]byte + var x11555 [1 << 17]byte + var x11556 [1 << 17]byte + var x11557 [1 << 17]byte + var x11558 [1 << 17]byte + var x11559 [1 << 17]byte + var x11560 [1 << 17]byte + var x11561 [1 << 17]byte + var x11562 [1 << 17]byte + var x11563 [1 << 17]byte + var x11564 [1 << 17]byte + var x11565 [1 << 17]byte + var x11566 [1 << 17]byte + var x11567 [1 << 17]byte + var x11568 [1 << 17]byte + var x11569 [1 << 17]byte + var x11570 [1 << 17]byte + var x11571 [1 << 17]byte + var x11572 [1 << 17]byte + var x11573 [1 << 17]byte + var x11574 [1 << 17]byte + var x11575 [1 << 17]byte + var x11576 [1 << 17]byte + var x11577 [1 << 17]byte + var x11578 [1 << 17]byte + var x11579 [1 << 17]byte + var x11580 [1 << 17]byte + var x11581 [1 << 17]byte + var x11582 [1 << 17]byte + var x11583 [1 << 17]byte + var x11584 [1 << 17]byte + var x11585 [1 << 17]byte + var x11586 [1 << 17]byte + var x11587 [1 << 17]byte + var x11588 [1 << 17]byte + var x11589 [1 << 17]byte + var x11590 [1 << 17]byte + var x11591 [1 << 17]byte + var x11592 [1 << 17]byte + var x11593 [1 << 17]byte + var x11594 [1 << 17]byte + var x11595 [1 << 17]byte + var x11596 [1 << 17]byte + var x11597 [1 << 17]byte + var x11598 [1 << 17]byte + var x11599 [1 << 17]byte + var x11600 [1 << 17]byte + var x11601 [1 << 17]byte + var x11602 [1 << 17]byte + var x11603 [1 << 17]byte + var x11604 [1 << 17]byte + var x11605 [1 << 17]byte + var x11606 [1 << 17]byte + var x11607 [1 << 17]byte + var x11608 [1 << 17]byte + var x11609 [1 << 17]byte + var x11610 [1 << 17]byte + var x11611 [1 << 17]byte + var x11612 [1 << 17]byte + var x11613 [1 << 17]byte + var x11614 [1 << 17]byte + var x11615 [1 << 17]byte + var x11616 [1 << 17]byte + var x11617 [1 << 17]byte + var x11618 [1 << 17]byte + var x11619 [1 << 17]byte + var x11620 [1 << 17]byte + var x11621 [1 << 17]byte + var x11622 [1 << 17]byte + var x11623 [1 << 17]byte + var x11624 [1 << 17]byte + var x11625 [1 << 17]byte + var x11626 [1 << 17]byte + var x11627 [1 << 17]byte + var x11628 [1 << 17]byte + var x11629 [1 << 17]byte + var x11630 [1 << 17]byte + var x11631 [1 << 17]byte + var x11632 [1 << 17]byte + var x11633 [1 << 17]byte + var x11634 [1 << 17]byte + var x11635 [1 << 17]byte + var x11636 [1 << 17]byte + var x11637 [1 << 17]byte + var x11638 [1 << 17]byte + var x11639 [1 << 17]byte + var x11640 [1 << 17]byte + var x11641 [1 << 17]byte + var x11642 [1 << 17]byte + var x11643 [1 << 17]byte + var x11644 [1 << 17]byte + var x11645 [1 << 17]byte + var x11646 [1 << 17]byte + var x11647 [1 << 17]byte + var x11648 [1 << 17]byte + var x11649 [1 << 17]byte + var x11650 [1 << 17]byte + var x11651 [1 << 17]byte + var x11652 [1 << 17]byte + var x11653 [1 << 17]byte + var x11654 [1 << 17]byte + var x11655 [1 << 17]byte + var x11656 [1 << 17]byte + var x11657 [1 << 17]byte + var x11658 [1 << 17]byte + var x11659 [1 << 17]byte + var x11660 [1 << 17]byte + var x11661 [1 << 17]byte + var x11662 [1 << 17]byte + var x11663 [1 << 17]byte + var x11664 [1 << 17]byte + var x11665 [1 << 17]byte + var x11666 [1 << 17]byte + var x11667 [1 << 17]byte + var x11668 [1 << 17]byte + var x11669 [1 << 17]byte + var x11670 [1 << 17]byte + var x11671 [1 << 17]byte + var x11672 [1 << 17]byte + var x11673 [1 << 17]byte + var x11674 [1 << 17]byte + var x11675 [1 << 17]byte + var x11676 [1 << 17]byte + var x11677 [1 << 17]byte + var x11678 [1 << 17]byte + var x11679 [1 << 17]byte + var x11680 [1 << 17]byte + var x11681 [1 << 17]byte + var x11682 [1 << 17]byte + var x11683 [1 << 17]byte + var x11684 [1 << 17]byte + var x11685 [1 << 17]byte + var x11686 [1 << 17]byte + var x11687 [1 << 17]byte + var x11688 [1 << 17]byte + var x11689 [1 << 17]byte + var x11690 [1 << 17]byte + var x11691 [1 << 17]byte + var x11692 [1 << 17]byte + var x11693 [1 << 17]byte + var x11694 [1 << 17]byte + var x11695 [1 << 17]byte + var x11696 [1 << 17]byte + var x11697 [1 << 17]byte + var x11698 [1 << 17]byte + var x11699 [1 << 17]byte + var x11700 [1 << 17]byte + var x11701 [1 << 17]byte + var x11702 [1 << 17]byte + var x11703 [1 << 17]byte + var x11704 [1 << 17]byte + var x11705 [1 << 17]byte + var x11706 [1 << 17]byte + var x11707 [1 << 17]byte + var x11708 [1 << 17]byte + var x11709 [1 << 17]byte + var x11710 [1 << 17]byte + var x11711 [1 << 17]byte + var x11712 [1 << 17]byte + var x11713 [1 << 17]byte + var x11714 [1 << 17]byte + var x11715 [1 << 17]byte + var x11716 [1 << 17]byte + var x11717 [1 << 17]byte + var x11718 [1 << 17]byte + var x11719 [1 << 17]byte + var x11720 [1 << 17]byte + var x11721 [1 << 17]byte + var x11722 [1 << 17]byte + var x11723 [1 << 17]byte + var x11724 [1 << 17]byte + var x11725 [1 << 17]byte + var x11726 [1 << 17]byte + var x11727 [1 << 17]byte + var x11728 [1 << 17]byte + var x11729 [1 << 17]byte + var x11730 [1 << 17]byte + var x11731 [1 << 17]byte + var x11732 [1 << 17]byte + var x11733 [1 << 17]byte + var x11734 [1 << 17]byte + var x11735 [1 << 17]byte + var x11736 [1 << 17]byte + var x11737 [1 << 17]byte + var x11738 [1 << 17]byte + var x11739 [1 << 17]byte + var x11740 [1 << 17]byte + var x11741 [1 << 17]byte + var x11742 [1 << 17]byte + var x11743 [1 << 17]byte + var x11744 [1 << 17]byte + var x11745 [1 << 17]byte + var x11746 [1 << 17]byte + var x11747 [1 << 17]byte + var x11748 [1 << 17]byte + var x11749 [1 << 17]byte + var x11750 [1 << 17]byte + var x11751 [1 << 17]byte + var x11752 [1 << 17]byte + var x11753 [1 << 17]byte + var x11754 [1 << 17]byte + var x11755 [1 << 17]byte + var x11756 [1 << 17]byte + var x11757 [1 << 17]byte + var x11758 [1 << 17]byte + var x11759 [1 << 17]byte + var x11760 [1 << 17]byte + var x11761 [1 << 17]byte + var x11762 [1 << 17]byte + var x11763 [1 << 17]byte + var x11764 [1 << 17]byte + var x11765 [1 << 17]byte + var x11766 [1 << 17]byte + var x11767 [1 << 17]byte + var x11768 [1 << 17]byte + var x11769 [1 << 17]byte + var x11770 [1 << 17]byte + var x11771 [1 << 17]byte + var x11772 [1 << 17]byte + var x11773 [1 << 17]byte + var x11774 [1 << 17]byte + var x11775 [1 << 17]byte + var x11776 [1 << 17]byte + var x11777 [1 << 17]byte + var x11778 [1 << 17]byte + var x11779 [1 << 17]byte + var x11780 [1 << 17]byte + var x11781 [1 << 17]byte + var x11782 [1 << 17]byte + var x11783 [1 << 17]byte + var x11784 [1 << 17]byte + var x11785 [1 << 17]byte + var x11786 [1 << 17]byte + var x11787 [1 << 17]byte + var x11788 [1 << 17]byte + var x11789 [1 << 17]byte + var x11790 [1 << 17]byte + var x11791 [1 << 17]byte + var x11792 [1 << 17]byte + var x11793 [1 << 17]byte + var x11794 [1 << 17]byte + var x11795 [1 << 17]byte + var x11796 [1 << 17]byte + var x11797 [1 << 17]byte + var x11798 [1 << 17]byte + var x11799 [1 << 17]byte + var x11800 [1 << 17]byte + var x11801 [1 << 17]byte + var x11802 [1 << 17]byte + var x11803 [1 << 17]byte + var x11804 [1 << 17]byte + var x11805 [1 << 17]byte + var x11806 [1 << 17]byte + var x11807 [1 << 17]byte + var x11808 [1 << 17]byte + var x11809 [1 << 17]byte + var x11810 [1 << 17]byte + var x11811 [1 << 17]byte + var x11812 [1 << 17]byte + var x11813 [1 << 17]byte + var x11814 [1 << 17]byte + var x11815 [1 << 17]byte + var x11816 [1 << 17]byte + var x11817 [1 << 17]byte + var x11818 [1 << 17]byte + var x11819 [1 << 17]byte + var x11820 [1 << 17]byte + var x11821 [1 << 17]byte + var x11822 [1 << 17]byte + var x11823 [1 << 17]byte + var x11824 [1 << 17]byte + var x11825 [1 << 17]byte + var x11826 [1 << 17]byte + var x11827 [1 << 17]byte + var x11828 [1 << 17]byte + var x11829 [1 << 17]byte + var x11830 [1 << 17]byte + var x11831 [1 << 17]byte + var x11832 [1 << 17]byte + var x11833 [1 << 17]byte + var x11834 [1 << 17]byte + var x11835 [1 << 17]byte + var x11836 [1 << 17]byte + var x11837 [1 << 17]byte + var x11838 [1 << 17]byte + var x11839 [1 << 17]byte + var x11840 [1 << 17]byte + var x11841 [1 << 17]byte + var x11842 [1 << 17]byte + var x11843 [1 << 17]byte + var x11844 [1 << 17]byte + var x11845 [1 << 17]byte + var x11846 [1 << 17]byte + var x11847 [1 << 17]byte + var x11848 [1 << 17]byte + var x11849 [1 << 17]byte + var x11850 [1 << 17]byte + var x11851 [1 << 17]byte + var x11852 [1 << 17]byte + var x11853 [1 << 17]byte + var x11854 [1 << 17]byte + var x11855 [1 << 17]byte + var x11856 [1 << 17]byte + var x11857 [1 << 17]byte + var x11858 [1 << 17]byte + var x11859 [1 << 17]byte + var x11860 [1 << 17]byte + var x11861 [1 << 17]byte + var x11862 [1 << 17]byte + var x11863 [1 << 17]byte + var x11864 [1 << 17]byte + var x11865 [1 << 17]byte + var x11866 [1 << 17]byte + var x11867 [1 << 17]byte + var x11868 [1 << 17]byte + var x11869 [1 << 17]byte + var x11870 [1 << 17]byte + var x11871 [1 << 17]byte + var x11872 [1 << 17]byte + var x11873 [1 << 17]byte + var x11874 [1 << 17]byte + var x11875 [1 << 17]byte + var x11876 [1 << 17]byte + var x11877 [1 << 17]byte + var x11878 [1 << 17]byte + var x11879 [1 << 17]byte + var x11880 [1 << 17]byte + var x11881 [1 << 17]byte + var x11882 [1 << 17]byte + var x11883 [1 << 17]byte + var x11884 [1 << 17]byte + var x11885 [1 << 17]byte + var x11886 [1 << 17]byte + var x11887 [1 << 17]byte + var x11888 [1 << 17]byte + var x11889 [1 << 17]byte + var x11890 [1 << 17]byte + var x11891 [1 << 17]byte + var x11892 [1 << 17]byte + var x11893 [1 << 17]byte + var x11894 [1 << 17]byte + var x11895 [1 << 17]byte + var x11896 [1 << 17]byte + var x11897 [1 << 17]byte + var x11898 [1 << 17]byte + var x11899 [1 << 17]byte + var x11900 [1 << 17]byte + var x11901 [1 << 17]byte + var x11902 [1 << 17]byte + var x11903 [1 << 17]byte + var x11904 [1 << 17]byte + var x11905 [1 << 17]byte + var x11906 [1 << 17]byte + var x11907 [1 << 17]byte + var x11908 [1 << 17]byte + var x11909 [1 << 17]byte + var x11910 [1 << 17]byte + var x11911 [1 << 17]byte + var x11912 [1 << 17]byte + var x11913 [1 << 17]byte + var x11914 [1 << 17]byte + var x11915 [1 << 17]byte + var x11916 [1 << 17]byte + var x11917 [1 << 17]byte + var x11918 [1 << 17]byte + var x11919 [1 << 17]byte + var x11920 [1 << 17]byte + var x11921 [1 << 17]byte + var x11922 [1 << 17]byte + var x11923 [1 << 17]byte + var x11924 [1 << 17]byte + var x11925 [1 << 17]byte + var x11926 [1 << 17]byte + var x11927 [1 << 17]byte + var x11928 [1 << 17]byte + var x11929 [1 << 17]byte + var x11930 [1 << 17]byte + var x11931 [1 << 17]byte + var x11932 [1 << 17]byte + var x11933 [1 << 17]byte + var x11934 [1 << 17]byte + var x11935 [1 << 17]byte + var x11936 [1 << 17]byte + var x11937 [1 << 17]byte + var x11938 [1 << 17]byte + var x11939 [1 << 17]byte + var x11940 [1 << 17]byte + var x11941 [1 << 17]byte + var x11942 [1 << 17]byte + var x11943 [1 << 17]byte + var x11944 [1 << 17]byte + var x11945 [1 << 17]byte + var x11946 [1 << 17]byte + var x11947 [1 << 17]byte + var x11948 [1 << 17]byte + var x11949 [1 << 17]byte + var x11950 [1 << 17]byte + var x11951 [1 << 17]byte + var x11952 [1 << 17]byte + var x11953 [1 << 17]byte + var x11954 [1 << 17]byte + var x11955 [1 << 17]byte + var x11956 [1 << 17]byte + var x11957 [1 << 17]byte + var x11958 [1 << 17]byte + var x11959 [1 << 17]byte + var x11960 [1 << 17]byte + var x11961 [1 << 17]byte + var x11962 [1 << 17]byte + var x11963 [1 << 17]byte + var x11964 [1 << 17]byte + var x11965 [1 << 17]byte + var x11966 [1 << 17]byte + var x11967 [1 << 17]byte + var x11968 [1 << 17]byte + var x11969 [1 << 17]byte + var x11970 [1 << 17]byte + var x11971 [1 << 17]byte + var x11972 [1 << 17]byte + var x11973 [1 << 17]byte + var x11974 [1 << 17]byte + var x11975 [1 << 17]byte + var x11976 [1 << 17]byte + var x11977 [1 << 17]byte + var x11978 [1 << 17]byte + var x11979 [1 << 17]byte + var x11980 [1 << 17]byte + var x11981 [1 << 17]byte + var x11982 [1 << 17]byte + var x11983 [1 << 17]byte + var x11984 [1 << 17]byte + var x11985 [1 << 17]byte + var x11986 [1 << 17]byte + var x11987 [1 << 17]byte + var x11988 [1 << 17]byte + var x11989 [1 << 17]byte + var x11990 [1 << 17]byte + var x11991 [1 << 17]byte + var x11992 [1 << 17]byte + var x11993 [1 << 17]byte + var x11994 [1 << 17]byte + var x11995 [1 << 17]byte + var x11996 [1 << 17]byte + var x11997 [1 << 17]byte + var x11998 [1 << 17]byte + var x11999 [1 << 17]byte + var x12000 [1 << 17]byte + var x12001 [1 << 17]byte + var x12002 [1 << 17]byte + var x12003 [1 << 17]byte + var x12004 [1 << 17]byte + var x12005 [1 << 17]byte + var x12006 [1 << 17]byte + var x12007 [1 << 17]byte + var x12008 [1 << 17]byte + var x12009 [1 << 17]byte + var x12010 [1 << 17]byte + var x12011 [1 << 17]byte + var x12012 [1 << 17]byte + var x12013 [1 << 17]byte + var x12014 [1 << 17]byte + var x12015 [1 << 17]byte + var x12016 [1 << 17]byte + var x12017 [1 << 17]byte + var x12018 [1 << 17]byte + var x12019 [1 << 17]byte + var x12020 [1 << 17]byte + var x12021 [1 << 17]byte + var x12022 [1 << 17]byte + var x12023 [1 << 17]byte + var x12024 [1 << 17]byte + var x12025 [1 << 17]byte + var x12026 [1 << 17]byte + var x12027 [1 << 17]byte + var x12028 [1 << 17]byte + var x12029 [1 << 17]byte + var x12030 [1 << 17]byte + var x12031 [1 << 17]byte + var x12032 [1 << 17]byte + var x12033 [1 << 17]byte + var x12034 [1 << 17]byte + var x12035 [1 << 17]byte + var x12036 [1 << 17]byte + var x12037 [1 << 17]byte + var x12038 [1 << 17]byte + var x12039 [1 << 17]byte + var x12040 [1 << 17]byte + var x12041 [1 << 17]byte + var x12042 [1 << 17]byte + var x12043 [1 << 17]byte + var x12044 [1 << 17]byte + var x12045 [1 << 17]byte + var x12046 [1 << 17]byte + var x12047 [1 << 17]byte + var x12048 [1 << 17]byte + var x12049 [1 << 17]byte + var x12050 [1 << 17]byte + var x12051 [1 << 17]byte + var x12052 [1 << 17]byte + var x12053 [1 << 17]byte + var x12054 [1 << 17]byte + var x12055 [1 << 17]byte + var x12056 [1 << 17]byte + var x12057 [1 << 17]byte + var x12058 [1 << 17]byte + var x12059 [1 << 17]byte + var x12060 [1 << 17]byte + var x12061 [1 << 17]byte + var x12062 [1 << 17]byte + var x12063 [1 << 17]byte + var x12064 [1 << 17]byte + var x12065 [1 << 17]byte + var x12066 [1 << 17]byte + var x12067 [1 << 17]byte + var x12068 [1 << 17]byte + var x12069 [1 << 17]byte + var x12070 [1 << 17]byte + var x12071 [1 << 17]byte + var x12072 [1 << 17]byte + var x12073 [1 << 17]byte + var x12074 [1 << 17]byte + var x12075 [1 << 17]byte + var x12076 [1 << 17]byte + var x12077 [1 << 17]byte + var x12078 [1 << 17]byte + var x12079 [1 << 17]byte + var x12080 [1 << 17]byte + var x12081 [1 << 17]byte + var x12082 [1 << 17]byte + var x12083 [1 << 17]byte + var x12084 [1 << 17]byte + var x12085 [1 << 17]byte + var x12086 [1 << 17]byte + var x12087 [1 << 17]byte + var x12088 [1 << 17]byte + var x12089 [1 << 17]byte + var x12090 [1 << 17]byte + var x12091 [1 << 17]byte + var x12092 [1 << 17]byte + var x12093 [1 << 17]byte + var x12094 [1 << 17]byte + var x12095 [1 << 17]byte + var x12096 [1 << 17]byte + var x12097 [1 << 17]byte + var x12098 [1 << 17]byte + var x12099 [1 << 17]byte + var x12100 [1 << 17]byte + var x12101 [1 << 17]byte + var x12102 [1 << 17]byte + var x12103 [1 << 17]byte + var x12104 [1 << 17]byte + var x12105 [1 << 17]byte + var x12106 [1 << 17]byte + var x12107 [1 << 17]byte + var x12108 [1 << 17]byte + var x12109 [1 << 17]byte + var x12110 [1 << 17]byte + var x12111 [1 << 17]byte + var x12112 [1 << 17]byte + var x12113 [1 << 17]byte + var x12114 [1 << 17]byte + var x12115 [1 << 17]byte + var x12116 [1 << 17]byte + var x12117 [1 << 17]byte + var x12118 [1 << 17]byte + var x12119 [1 << 17]byte + var x12120 [1 << 17]byte + var x12121 [1 << 17]byte + var x12122 [1 << 17]byte + var x12123 [1 << 17]byte + var x12124 [1 << 17]byte + var x12125 [1 << 17]byte + var x12126 [1 << 17]byte + var x12127 [1 << 17]byte + var x12128 [1 << 17]byte + var x12129 [1 << 17]byte + var x12130 [1 << 17]byte + var x12131 [1 << 17]byte + var x12132 [1 << 17]byte + var x12133 [1 << 17]byte + var x12134 [1 << 17]byte + var x12135 [1 << 17]byte + var x12136 [1 << 17]byte + var x12137 [1 << 17]byte + var x12138 [1 << 17]byte + var x12139 [1 << 17]byte + var x12140 [1 << 17]byte + var x12141 [1 << 17]byte + var x12142 [1 << 17]byte + var x12143 [1 << 17]byte + var x12144 [1 << 17]byte + var x12145 [1 << 17]byte + var x12146 [1 << 17]byte + var x12147 [1 << 17]byte + var x12148 [1 << 17]byte + var x12149 [1 << 17]byte + var x12150 [1 << 17]byte + var x12151 [1 << 17]byte + var x12152 [1 << 17]byte + var x12153 [1 << 17]byte + var x12154 [1 << 17]byte + var x12155 [1 << 17]byte + var x12156 [1 << 17]byte + var x12157 [1 << 17]byte + var x12158 [1 << 17]byte + var x12159 [1 << 17]byte + var x12160 [1 << 17]byte + var x12161 [1 << 17]byte + var x12162 [1 << 17]byte + var x12163 [1 << 17]byte + var x12164 [1 << 17]byte + var x12165 [1 << 17]byte + var x12166 [1 << 17]byte + var x12167 [1 << 17]byte + var x12168 [1 << 17]byte + var x12169 [1 << 17]byte + var x12170 [1 << 17]byte + var x12171 [1 << 17]byte + var x12172 [1 << 17]byte + var x12173 [1 << 17]byte + var x12174 [1 << 17]byte + var x12175 [1 << 17]byte + var x12176 [1 << 17]byte + var x12177 [1 << 17]byte + var x12178 [1 << 17]byte + var x12179 [1 << 17]byte + var x12180 [1 << 17]byte + var x12181 [1 << 17]byte + var x12182 [1 << 17]byte + var x12183 [1 << 17]byte + var x12184 [1 << 17]byte + var x12185 [1 << 17]byte + var x12186 [1 << 17]byte + var x12187 [1 << 17]byte + var x12188 [1 << 17]byte + var x12189 [1 << 17]byte + var x12190 [1 << 17]byte + var x12191 [1 << 17]byte + var x12192 [1 << 17]byte + var x12193 [1 << 17]byte + var x12194 [1 << 17]byte + var x12195 [1 << 17]byte + var x12196 [1 << 17]byte + var x12197 [1 << 17]byte + var x12198 [1 << 17]byte + var x12199 [1 << 17]byte + var x12200 [1 << 17]byte + var x12201 [1 << 17]byte + var x12202 [1 << 17]byte + var x12203 [1 << 17]byte + var x12204 [1 << 17]byte + var x12205 [1 << 17]byte + var x12206 [1 << 17]byte + var x12207 [1 << 17]byte + var x12208 [1 << 17]byte + var x12209 [1 << 17]byte + var x12210 [1 << 17]byte + var x12211 [1 << 17]byte + var x12212 [1 << 17]byte + var x12213 [1 << 17]byte + var x12214 [1 << 17]byte + var x12215 [1 << 17]byte + var x12216 [1 << 17]byte + var x12217 [1 << 17]byte + var x12218 [1 << 17]byte + var x12219 [1 << 17]byte + var x12220 [1 << 17]byte + var x12221 [1 << 17]byte + var x12222 [1 << 17]byte + var x12223 [1 << 17]byte + var x12224 [1 << 17]byte + var x12225 [1 << 17]byte + var x12226 [1 << 17]byte + var x12227 [1 << 17]byte + var x12228 [1 << 17]byte + var x12229 [1 << 17]byte + var x12230 [1 << 17]byte + var x12231 [1 << 17]byte + var x12232 [1 << 17]byte + var x12233 [1 << 17]byte + var x12234 [1 << 17]byte + var x12235 [1 << 17]byte + var x12236 [1 << 17]byte + var x12237 [1 << 17]byte + var x12238 [1 << 17]byte + var x12239 [1 << 17]byte + var x12240 [1 << 17]byte + var x12241 [1 << 17]byte + var x12242 [1 << 17]byte + var x12243 [1 << 17]byte + var x12244 [1 << 17]byte + var x12245 [1 << 17]byte + var x12246 [1 << 17]byte + var x12247 [1 << 17]byte + var x12248 [1 << 17]byte + var x12249 [1 << 17]byte + var x12250 [1 << 17]byte + var x12251 [1 << 17]byte + var x12252 [1 << 17]byte + var x12253 [1 << 17]byte + var x12254 [1 << 17]byte + var x12255 [1 << 17]byte + var x12256 [1 << 17]byte + var x12257 [1 << 17]byte + var x12258 [1 << 17]byte + var x12259 [1 << 17]byte + var x12260 [1 << 17]byte + var x12261 [1 << 17]byte + var x12262 [1 << 17]byte + var x12263 [1 << 17]byte + var x12264 [1 << 17]byte + var x12265 [1 << 17]byte + var x12266 [1 << 17]byte + var x12267 [1 << 17]byte + var x12268 [1 << 17]byte + var x12269 [1 << 17]byte + var x12270 [1 << 17]byte + var x12271 [1 << 17]byte + var x12272 [1 << 17]byte + var x12273 [1 << 17]byte + var x12274 [1 << 17]byte + var x12275 [1 << 17]byte + var x12276 [1 << 17]byte + var x12277 [1 << 17]byte + var x12278 [1 << 17]byte + var x12279 [1 << 17]byte + var x12280 [1 << 17]byte + var x12281 [1 << 17]byte + var x12282 [1 << 17]byte + var x12283 [1 << 17]byte + var x12284 [1 << 17]byte + var x12285 [1 << 17]byte + var x12286 [1 << 17]byte + var x12287 [1 << 17]byte + var x12288 [1 << 17]byte + var x12289 [1 << 17]byte + var x12290 [1 << 17]byte + var x12291 [1 << 17]byte + var x12292 [1 << 17]byte + var x12293 [1 << 17]byte + var x12294 [1 << 17]byte + var x12295 [1 << 17]byte + var x12296 [1 << 17]byte + var x12297 [1 << 17]byte + var x12298 [1 << 17]byte + var x12299 [1 << 17]byte + var x12300 [1 << 17]byte + var x12301 [1 << 17]byte + var x12302 [1 << 17]byte + var x12303 [1 << 17]byte + var x12304 [1 << 17]byte + var x12305 [1 << 17]byte + var x12306 [1 << 17]byte + var x12307 [1 << 17]byte + var x12308 [1 << 17]byte + var x12309 [1 << 17]byte + var x12310 [1 << 17]byte + var x12311 [1 << 17]byte + var x12312 [1 << 17]byte + var x12313 [1 << 17]byte + var x12314 [1 << 17]byte + var x12315 [1 << 17]byte + var x12316 [1 << 17]byte + var x12317 [1 << 17]byte + var x12318 [1 << 17]byte + var x12319 [1 << 17]byte + var x12320 [1 << 17]byte + var x12321 [1 << 17]byte + var x12322 [1 << 17]byte + var x12323 [1 << 17]byte + var x12324 [1 << 17]byte + var x12325 [1 << 17]byte + var x12326 [1 << 17]byte + var x12327 [1 << 17]byte + var x12328 [1 << 17]byte + var x12329 [1 << 17]byte + var x12330 [1 << 17]byte + var x12331 [1 << 17]byte + var x12332 [1 << 17]byte + var x12333 [1 << 17]byte + var x12334 [1 << 17]byte + var x12335 [1 << 17]byte + var x12336 [1 << 17]byte + var x12337 [1 << 17]byte + var x12338 [1 << 17]byte + var x12339 [1 << 17]byte + var x12340 [1 << 17]byte + var x12341 [1 << 17]byte + var x12342 [1 << 17]byte + var x12343 [1 << 17]byte + var x12344 [1 << 17]byte + var x12345 [1 << 17]byte + var x12346 [1 << 17]byte + var x12347 [1 << 17]byte + var x12348 [1 << 17]byte + var x12349 [1 << 17]byte + var x12350 [1 << 17]byte + var x12351 [1 << 17]byte + var x12352 [1 << 17]byte + var x12353 [1 << 17]byte + var x12354 [1 << 17]byte + var x12355 [1 << 17]byte + var x12356 [1 << 17]byte + var x12357 [1 << 17]byte + var x12358 [1 << 17]byte + var x12359 [1 << 17]byte + var x12360 [1 << 17]byte + var x12361 [1 << 17]byte + var x12362 [1 << 17]byte + var x12363 [1 << 17]byte + var x12364 [1 << 17]byte + var x12365 [1 << 17]byte + var x12366 [1 << 17]byte + var x12367 [1 << 17]byte + var x12368 [1 << 17]byte + var x12369 [1 << 17]byte + var x12370 [1 << 17]byte + var x12371 [1 << 17]byte + var x12372 [1 << 17]byte + var x12373 [1 << 17]byte + var x12374 [1 << 17]byte + var x12375 [1 << 17]byte + var x12376 [1 << 17]byte + var x12377 [1 << 17]byte + var x12378 [1 << 17]byte + var x12379 [1 << 17]byte + var x12380 [1 << 17]byte + var x12381 [1 << 17]byte + var x12382 [1 << 17]byte + var x12383 [1 << 17]byte + var x12384 [1 << 17]byte + var x12385 [1 << 17]byte + var x12386 [1 << 17]byte + var x12387 [1 << 17]byte + var x12388 [1 << 17]byte + var x12389 [1 << 17]byte + var x12390 [1 << 17]byte + var x12391 [1 << 17]byte + var x12392 [1 << 17]byte + var x12393 [1 << 17]byte + var x12394 [1 << 17]byte + var x12395 [1 << 17]byte + var x12396 [1 << 17]byte + var x12397 [1 << 17]byte + var x12398 [1 << 17]byte + var x12399 [1 << 17]byte + var x12400 [1 << 17]byte + var x12401 [1 << 17]byte + var x12402 [1 << 17]byte + var x12403 [1 << 17]byte + var x12404 [1 << 17]byte + var x12405 [1 << 17]byte + var x12406 [1 << 17]byte + var x12407 [1 << 17]byte + var x12408 [1 << 17]byte + var x12409 [1 << 17]byte + var x12410 [1 << 17]byte + var x12411 [1 << 17]byte + var x12412 [1 << 17]byte + var x12413 [1 << 17]byte + var x12414 [1 << 17]byte + var x12415 [1 << 17]byte + var x12416 [1 << 17]byte + var x12417 [1 << 17]byte + var x12418 [1 << 17]byte + var x12419 [1 << 17]byte + var x12420 [1 << 17]byte + var x12421 [1 << 17]byte + var x12422 [1 << 17]byte + var x12423 [1 << 17]byte + var x12424 [1 << 17]byte + var x12425 [1 << 17]byte + var x12426 [1 << 17]byte + var x12427 [1 << 17]byte + var x12428 [1 << 17]byte + var x12429 [1 << 17]byte + var x12430 [1 << 17]byte + var x12431 [1 << 17]byte + var x12432 [1 << 17]byte + var x12433 [1 << 17]byte + var x12434 [1 << 17]byte + var x12435 [1 << 17]byte + var x12436 [1 << 17]byte + var x12437 [1 << 17]byte + var x12438 [1 << 17]byte + var x12439 [1 << 17]byte + var x12440 [1 << 17]byte + var x12441 [1 << 17]byte + var x12442 [1 << 17]byte + var x12443 [1 << 17]byte + var x12444 [1 << 17]byte + var x12445 [1 << 17]byte + var x12446 [1 << 17]byte + var x12447 [1 << 17]byte + var x12448 [1 << 17]byte + var x12449 [1 << 17]byte + var x12450 [1 << 17]byte + var x12451 [1 << 17]byte + var x12452 [1 << 17]byte + var x12453 [1 << 17]byte + var x12454 [1 << 17]byte + var x12455 [1 << 17]byte + var x12456 [1 << 17]byte + var x12457 [1 << 17]byte + var x12458 [1 << 17]byte + var x12459 [1 << 17]byte + var x12460 [1 << 17]byte + var x12461 [1 << 17]byte + var x12462 [1 << 17]byte + var x12463 [1 << 17]byte + var x12464 [1 << 17]byte + var x12465 [1 << 17]byte + var x12466 [1 << 17]byte + var x12467 [1 << 17]byte + var x12468 [1 << 17]byte + var x12469 [1 << 17]byte + var x12470 [1 << 17]byte + var x12471 [1 << 17]byte + var x12472 [1 << 17]byte + var x12473 [1 << 17]byte + var x12474 [1 << 17]byte + var x12475 [1 << 17]byte + var x12476 [1 << 17]byte + var x12477 [1 << 17]byte + var x12478 [1 << 17]byte + var x12479 [1 << 17]byte + var x12480 [1 << 17]byte + var x12481 [1 << 17]byte + var x12482 [1 << 17]byte + var x12483 [1 << 17]byte + var x12484 [1 << 17]byte + var x12485 [1 << 17]byte + var x12486 [1 << 17]byte + var x12487 [1 << 17]byte + var x12488 [1 << 17]byte + var x12489 [1 << 17]byte + var x12490 [1 << 17]byte + var x12491 [1 << 17]byte + var x12492 [1 << 17]byte + var x12493 [1 << 17]byte + var x12494 [1 << 17]byte + var x12495 [1 << 17]byte + var x12496 [1 << 17]byte + var x12497 [1 << 17]byte + var x12498 [1 << 17]byte + var x12499 [1 << 17]byte + var x12500 [1 << 17]byte + var x12501 [1 << 17]byte + var x12502 [1 << 17]byte + var x12503 [1 << 17]byte + var x12504 [1 << 17]byte + var x12505 [1 << 17]byte + var x12506 [1 << 17]byte + var x12507 [1 << 17]byte + var x12508 [1 << 17]byte + var x12509 [1 << 17]byte + var x12510 [1 << 17]byte + var x12511 [1 << 17]byte + var x12512 [1 << 17]byte + var x12513 [1 << 17]byte + var x12514 [1 << 17]byte + var x12515 [1 << 17]byte + var x12516 [1 << 17]byte + var x12517 [1 << 17]byte + var x12518 [1 << 17]byte + var x12519 [1 << 17]byte + var x12520 [1 << 17]byte + var x12521 [1 << 17]byte + var x12522 [1 << 17]byte + var x12523 [1 << 17]byte + var x12524 [1 << 17]byte + var x12525 [1 << 17]byte + var x12526 [1 << 17]byte + var x12527 [1 << 17]byte + var x12528 [1 << 17]byte + var x12529 [1 << 17]byte + var x12530 [1 << 17]byte + var x12531 [1 << 17]byte + var x12532 [1 << 17]byte + var x12533 [1 << 17]byte + var x12534 [1 << 17]byte + var x12535 [1 << 17]byte + var x12536 [1 << 17]byte + var x12537 [1 << 17]byte + var x12538 [1 << 17]byte + var x12539 [1 << 17]byte + var x12540 [1 << 17]byte + var x12541 [1 << 17]byte + var x12542 [1 << 17]byte + var x12543 [1 << 17]byte + var x12544 [1 << 17]byte + var x12545 [1 << 17]byte + var x12546 [1 << 17]byte + var x12547 [1 << 17]byte + var x12548 [1 << 17]byte + var x12549 [1 << 17]byte + var x12550 [1 << 17]byte + var x12551 [1 << 17]byte + var x12552 [1 << 17]byte + var x12553 [1 << 17]byte + var x12554 [1 << 17]byte + var x12555 [1 << 17]byte + var x12556 [1 << 17]byte + var x12557 [1 << 17]byte + var x12558 [1 << 17]byte + var x12559 [1 << 17]byte + var x12560 [1 << 17]byte + var x12561 [1 << 17]byte + var x12562 [1 << 17]byte + var x12563 [1 << 17]byte + var x12564 [1 << 17]byte + var x12565 [1 << 17]byte + var x12566 [1 << 17]byte + var x12567 [1 << 17]byte + var x12568 [1 << 17]byte + var x12569 [1 << 17]byte + var x12570 [1 << 17]byte + var x12571 [1 << 17]byte + var x12572 [1 << 17]byte + var x12573 [1 << 17]byte + var x12574 [1 << 17]byte + var x12575 [1 << 17]byte + var x12576 [1 << 17]byte + var x12577 [1 << 17]byte + var x12578 [1 << 17]byte + var x12579 [1 << 17]byte + var x12580 [1 << 17]byte + var x12581 [1 << 17]byte + var x12582 [1 << 17]byte + var x12583 [1 << 17]byte + var x12584 [1 << 17]byte + var x12585 [1 << 17]byte + var x12586 [1 << 17]byte + var x12587 [1 << 17]byte + var x12588 [1 << 17]byte + var x12589 [1 << 17]byte + var x12590 [1 << 17]byte + var x12591 [1 << 17]byte + var x12592 [1 << 17]byte + var x12593 [1 << 17]byte + var x12594 [1 << 17]byte + var x12595 [1 << 17]byte + var x12596 [1 << 17]byte + var x12597 [1 << 17]byte + var x12598 [1 << 17]byte + var x12599 [1 << 17]byte + var x12600 [1 << 17]byte + var x12601 [1 << 17]byte + var x12602 [1 << 17]byte + var x12603 [1 << 17]byte + var x12604 [1 << 17]byte + var x12605 [1 << 17]byte + var x12606 [1 << 17]byte + var x12607 [1 << 17]byte + var x12608 [1 << 17]byte + var x12609 [1 << 17]byte + var x12610 [1 << 17]byte + var x12611 [1 << 17]byte + var x12612 [1 << 17]byte + var x12613 [1 << 17]byte + var x12614 [1 << 17]byte + var x12615 [1 << 17]byte + var x12616 [1 << 17]byte + var x12617 [1 << 17]byte + var x12618 [1 << 17]byte + var x12619 [1 << 17]byte + var x12620 [1 << 17]byte + var x12621 [1 << 17]byte + var x12622 [1 << 17]byte + var x12623 [1 << 17]byte + var x12624 [1 << 17]byte + var x12625 [1 << 17]byte + var x12626 [1 << 17]byte + var x12627 [1 << 17]byte + var x12628 [1 << 17]byte + var x12629 [1 << 17]byte + var x12630 [1 << 17]byte + var x12631 [1 << 17]byte + var x12632 [1 << 17]byte + var x12633 [1 << 17]byte + var x12634 [1 << 17]byte + var x12635 [1 << 17]byte + var x12636 [1 << 17]byte + var x12637 [1 << 17]byte + var x12638 [1 << 17]byte + var x12639 [1 << 17]byte + var x12640 [1 << 17]byte + var x12641 [1 << 17]byte + var x12642 [1 << 17]byte + var x12643 [1 << 17]byte + var x12644 [1 << 17]byte + var x12645 [1 << 17]byte + var x12646 [1 << 17]byte + var x12647 [1 << 17]byte + var x12648 [1 << 17]byte + var x12649 [1 << 17]byte + var x12650 [1 << 17]byte + var x12651 [1 << 17]byte + var x12652 [1 << 17]byte + var x12653 [1 << 17]byte + var x12654 [1 << 17]byte + var x12655 [1 << 17]byte + var x12656 [1 << 17]byte + var x12657 [1 << 17]byte + var x12658 [1 << 17]byte + var x12659 [1 << 17]byte + var x12660 [1 << 17]byte + var x12661 [1 << 17]byte + var x12662 [1 << 17]byte + var x12663 [1 << 17]byte + var x12664 [1 << 17]byte + var x12665 [1 << 17]byte + var x12666 [1 << 17]byte + var x12667 [1 << 17]byte + var x12668 [1 << 17]byte + var x12669 [1 << 17]byte + var x12670 [1 << 17]byte + var x12671 [1 << 17]byte + var x12672 [1 << 17]byte + var x12673 [1 << 17]byte + var x12674 [1 << 17]byte + var x12675 [1 << 17]byte + var x12676 [1 << 17]byte + var x12677 [1 << 17]byte + var x12678 [1 << 17]byte + var x12679 [1 << 17]byte + var x12680 [1 << 17]byte + var x12681 [1 << 17]byte + var x12682 [1 << 17]byte + var x12683 [1 << 17]byte + var x12684 [1 << 17]byte + var x12685 [1 << 17]byte + var x12686 [1 << 17]byte + var x12687 [1 << 17]byte + var x12688 [1 << 17]byte + var x12689 [1 << 17]byte + var x12690 [1 << 17]byte + var x12691 [1 << 17]byte + var x12692 [1 << 17]byte + var x12693 [1 << 17]byte + var x12694 [1 << 17]byte + var x12695 [1 << 17]byte + var x12696 [1 << 17]byte + var x12697 [1 << 17]byte + var x12698 [1 << 17]byte + var x12699 [1 << 17]byte + var x12700 [1 << 17]byte + var x12701 [1 << 17]byte + var x12702 [1 << 17]byte + var x12703 [1 << 17]byte + var x12704 [1 << 17]byte + var x12705 [1 << 17]byte + var x12706 [1 << 17]byte + var x12707 [1 << 17]byte + var x12708 [1 << 17]byte + var x12709 [1 << 17]byte + var x12710 [1 << 17]byte + var x12711 [1 << 17]byte + var x12712 [1 << 17]byte + var x12713 [1 << 17]byte + var x12714 [1 << 17]byte + var x12715 [1 << 17]byte + var x12716 [1 << 17]byte + var x12717 [1 << 17]byte + var x12718 [1 << 17]byte + var x12719 [1 << 17]byte + var x12720 [1 << 17]byte + var x12721 [1 << 17]byte + var x12722 [1 << 17]byte + var x12723 [1 << 17]byte + var x12724 [1 << 17]byte + var x12725 [1 << 17]byte + var x12726 [1 << 17]byte + var x12727 [1 << 17]byte + var x12728 [1 << 17]byte + var x12729 [1 << 17]byte + var x12730 [1 << 17]byte + var x12731 [1 << 17]byte + var x12732 [1 << 17]byte + var x12733 [1 << 17]byte + var x12734 [1 << 17]byte + var x12735 [1 << 17]byte + var x12736 [1 << 17]byte + var x12737 [1 << 17]byte + var x12738 [1 << 17]byte + var x12739 [1 << 17]byte + var x12740 [1 << 17]byte + var x12741 [1 << 17]byte + var x12742 [1 << 17]byte + var x12743 [1 << 17]byte + var x12744 [1 << 17]byte + var x12745 [1 << 17]byte + var x12746 [1 << 17]byte + var x12747 [1 << 17]byte + var x12748 [1 << 17]byte + var x12749 [1 << 17]byte + var x12750 [1 << 17]byte + var x12751 [1 << 17]byte + var x12752 [1 << 17]byte + var x12753 [1 << 17]byte + var x12754 [1 << 17]byte + var x12755 [1 << 17]byte + var x12756 [1 << 17]byte + var x12757 [1 << 17]byte + var x12758 [1 << 17]byte + var x12759 [1 << 17]byte + var x12760 [1 << 17]byte + var x12761 [1 << 17]byte + var x12762 [1 << 17]byte + var x12763 [1 << 17]byte + var x12764 [1 << 17]byte + var x12765 [1 << 17]byte + var x12766 [1 << 17]byte + var x12767 [1 << 17]byte + var x12768 [1 << 17]byte + var x12769 [1 << 17]byte + var x12770 [1 << 17]byte + var x12771 [1 << 17]byte + var x12772 [1 << 17]byte + var x12773 [1 << 17]byte + var x12774 [1 << 17]byte + var x12775 [1 << 17]byte + var x12776 [1 << 17]byte + var x12777 [1 << 17]byte + var x12778 [1 << 17]byte + var x12779 [1 << 17]byte + var x12780 [1 << 17]byte + var x12781 [1 << 17]byte + var x12782 [1 << 17]byte + var x12783 [1 << 17]byte + var x12784 [1 << 17]byte + var x12785 [1 << 17]byte + var x12786 [1 << 17]byte + var x12787 [1 << 17]byte + var x12788 [1 << 17]byte + var x12789 [1 << 17]byte + var x12790 [1 << 17]byte + var x12791 [1 << 17]byte + var x12792 [1 << 17]byte + var x12793 [1 << 17]byte + var x12794 [1 << 17]byte + var x12795 [1 << 17]byte + var x12796 [1 << 17]byte + var x12797 [1 << 17]byte + var x12798 [1 << 17]byte + var x12799 [1 << 17]byte + var x12800 [1 << 17]byte + var x12801 [1 << 17]byte + var x12802 [1 << 17]byte + var x12803 [1 << 17]byte + var x12804 [1 << 17]byte + var x12805 [1 << 17]byte + var x12806 [1 << 17]byte + var x12807 [1 << 17]byte + var x12808 [1 << 17]byte + var x12809 [1 << 17]byte + var x12810 [1 << 17]byte + var x12811 [1 << 17]byte + var x12812 [1 << 17]byte + var x12813 [1 << 17]byte + var x12814 [1 << 17]byte + var x12815 [1 << 17]byte + var x12816 [1 << 17]byte + var x12817 [1 << 17]byte + var x12818 [1 << 17]byte + var x12819 [1 << 17]byte + var x12820 [1 << 17]byte + var x12821 [1 << 17]byte + var x12822 [1 << 17]byte + var x12823 [1 << 17]byte + var x12824 [1 << 17]byte + var x12825 [1 << 17]byte + var x12826 [1 << 17]byte + var x12827 [1 << 17]byte + var x12828 [1 << 17]byte + var x12829 [1 << 17]byte + var x12830 [1 << 17]byte + var x12831 [1 << 17]byte + var x12832 [1 << 17]byte + var x12833 [1 << 17]byte + var x12834 [1 << 17]byte + var x12835 [1 << 17]byte + var x12836 [1 << 17]byte + var x12837 [1 << 17]byte + var x12838 [1 << 17]byte + var x12839 [1 << 17]byte + var x12840 [1 << 17]byte + var x12841 [1 << 17]byte + var x12842 [1 << 17]byte + var x12843 [1 << 17]byte + var x12844 [1 << 17]byte + var x12845 [1 << 17]byte + var x12846 [1 << 17]byte + var x12847 [1 << 17]byte + var x12848 [1 << 17]byte + var x12849 [1 << 17]byte + var x12850 [1 << 17]byte + var x12851 [1 << 17]byte + var x12852 [1 << 17]byte + var x12853 [1 << 17]byte + var x12854 [1 << 17]byte + var x12855 [1 << 17]byte + var x12856 [1 << 17]byte + var x12857 [1 << 17]byte + var x12858 [1 << 17]byte + var x12859 [1 << 17]byte + var x12860 [1 << 17]byte + var x12861 [1 << 17]byte + var x12862 [1 << 17]byte + var x12863 [1 << 17]byte + var x12864 [1 << 17]byte + var x12865 [1 << 17]byte + var x12866 [1 << 17]byte + var x12867 [1 << 17]byte + var x12868 [1 << 17]byte + var x12869 [1 << 17]byte + var x12870 [1 << 17]byte + var x12871 [1 << 17]byte + var x12872 [1 << 17]byte + var x12873 [1 << 17]byte + var x12874 [1 << 17]byte + var x12875 [1 << 17]byte + var x12876 [1 << 17]byte + var x12877 [1 << 17]byte + var x12878 [1 << 17]byte + var x12879 [1 << 17]byte + var x12880 [1 << 17]byte + var x12881 [1 << 17]byte + var x12882 [1 << 17]byte + var x12883 [1 << 17]byte + var x12884 [1 << 17]byte + var x12885 [1 << 17]byte + var x12886 [1 << 17]byte + var x12887 [1 << 17]byte + var x12888 [1 << 17]byte + var x12889 [1 << 17]byte + var x12890 [1 << 17]byte + var x12891 [1 << 17]byte + var x12892 [1 << 17]byte + var x12893 [1 << 17]byte + var x12894 [1 << 17]byte + var x12895 [1 << 17]byte + var x12896 [1 << 17]byte + var x12897 [1 << 17]byte + var x12898 [1 << 17]byte + var x12899 [1 << 17]byte + var x12900 [1 << 17]byte + var x12901 [1 << 17]byte + var x12902 [1 << 17]byte + var x12903 [1 << 17]byte + var x12904 [1 << 17]byte + var x12905 [1 << 17]byte + var x12906 [1 << 17]byte + var x12907 [1 << 17]byte + var x12908 [1 << 17]byte + var x12909 [1 << 17]byte + var x12910 [1 << 17]byte + var x12911 [1 << 17]byte + var x12912 [1 << 17]byte + var x12913 [1 << 17]byte + var x12914 [1 << 17]byte + var x12915 [1 << 17]byte + var x12916 [1 << 17]byte + var x12917 [1 << 17]byte + var x12918 [1 << 17]byte + var x12919 [1 << 17]byte + var x12920 [1 << 17]byte + var x12921 [1 << 17]byte + var x12922 [1 << 17]byte + var x12923 [1 << 17]byte + var x12924 [1 << 17]byte + var x12925 [1 << 17]byte + var x12926 [1 << 17]byte + var x12927 [1 << 17]byte + var x12928 [1 << 17]byte + var x12929 [1 << 17]byte + var x12930 [1 << 17]byte + var x12931 [1 << 17]byte + var x12932 [1 << 17]byte + var x12933 [1 << 17]byte + var x12934 [1 << 17]byte + var x12935 [1 << 17]byte + var x12936 [1 << 17]byte + var x12937 [1 << 17]byte + var x12938 [1 << 17]byte + var x12939 [1 << 17]byte + var x12940 [1 << 17]byte + var x12941 [1 << 17]byte + var x12942 [1 << 17]byte + var x12943 [1 << 17]byte + var x12944 [1 << 17]byte + var x12945 [1 << 17]byte + var x12946 [1 << 17]byte + var x12947 [1 << 17]byte + var x12948 [1 << 17]byte + var x12949 [1 << 17]byte + var x12950 [1 << 17]byte + var x12951 [1 << 17]byte + var x12952 [1 << 17]byte + var x12953 [1 << 17]byte + var x12954 [1 << 17]byte + var x12955 [1 << 17]byte + var x12956 [1 << 17]byte + var x12957 [1 << 17]byte + var x12958 [1 << 17]byte + var x12959 [1 << 17]byte + var x12960 [1 << 17]byte + var x12961 [1 << 17]byte + var x12962 [1 << 17]byte + var x12963 [1 << 17]byte + var x12964 [1 << 17]byte + var x12965 [1 << 17]byte + var x12966 [1 << 17]byte + var x12967 [1 << 17]byte + var x12968 [1 << 17]byte + var x12969 [1 << 17]byte + var x12970 [1 << 17]byte + var x12971 [1 << 17]byte + var x12972 [1 << 17]byte + var x12973 [1 << 17]byte + var x12974 [1 << 17]byte + var x12975 [1 << 17]byte + var x12976 [1 << 17]byte + var x12977 [1 << 17]byte + var x12978 [1 << 17]byte + var x12979 [1 << 17]byte + var x12980 [1 << 17]byte + var x12981 [1 << 17]byte + var x12982 [1 << 17]byte + var x12983 [1 << 17]byte + var x12984 [1 << 17]byte + var x12985 [1 << 17]byte + var x12986 [1 << 17]byte + var x12987 [1 << 17]byte + var x12988 [1 << 17]byte + var x12989 [1 << 17]byte + var x12990 [1 << 17]byte + var x12991 [1 << 17]byte + var x12992 [1 << 17]byte + var x12993 [1 << 17]byte + var x12994 [1 << 17]byte + var x12995 [1 << 17]byte + var x12996 [1 << 17]byte + var x12997 [1 << 17]byte + var x12998 [1 << 17]byte + var x12999 [1 << 17]byte + var x13000 [1 << 17]byte + var x13001 [1 << 17]byte + var x13002 [1 << 17]byte + var x13003 [1 << 17]byte + var x13004 [1 << 17]byte + var x13005 [1 << 17]byte + var x13006 [1 << 17]byte + var x13007 [1 << 17]byte + var x13008 [1 << 17]byte + var x13009 [1 << 17]byte + var x13010 [1 << 17]byte + var x13011 [1 << 17]byte + var x13012 [1 << 17]byte + var x13013 [1 << 17]byte + var x13014 [1 << 17]byte + var x13015 [1 << 17]byte + var x13016 [1 << 17]byte + var x13017 [1 << 17]byte + var x13018 [1 << 17]byte + var x13019 [1 << 17]byte + var x13020 [1 << 17]byte + var x13021 [1 << 17]byte + var x13022 [1 << 17]byte + var x13023 [1 << 17]byte + var x13024 [1 << 17]byte + var x13025 [1 << 17]byte + var x13026 [1 << 17]byte + var x13027 [1 << 17]byte + var x13028 [1 << 17]byte + var x13029 [1 << 17]byte + var x13030 [1 << 17]byte + var x13031 [1 << 17]byte + var x13032 [1 << 17]byte + var x13033 [1 << 17]byte + var x13034 [1 << 17]byte + var x13035 [1 << 17]byte + var x13036 [1 << 17]byte + var x13037 [1 << 17]byte + var x13038 [1 << 17]byte + var x13039 [1 << 17]byte + var x13040 [1 << 17]byte + var x13041 [1 << 17]byte + var x13042 [1 << 17]byte + var x13043 [1 << 17]byte + var x13044 [1 << 17]byte + var x13045 [1 << 17]byte + var x13046 [1 << 17]byte + var x13047 [1 << 17]byte + var x13048 [1 << 17]byte + var x13049 [1 << 17]byte + var x13050 [1 << 17]byte + var x13051 [1 << 17]byte + var x13052 [1 << 17]byte + var x13053 [1 << 17]byte + var x13054 [1 << 17]byte + var x13055 [1 << 17]byte + var x13056 [1 << 17]byte + var x13057 [1 << 17]byte + var x13058 [1 << 17]byte + var x13059 [1 << 17]byte + var x13060 [1 << 17]byte + var x13061 [1 << 17]byte + var x13062 [1 << 17]byte + var x13063 [1 << 17]byte + var x13064 [1 << 17]byte + var x13065 [1 << 17]byte + var x13066 [1 << 17]byte + var x13067 [1 << 17]byte + var x13068 [1 << 17]byte + var x13069 [1 << 17]byte + var x13070 [1 << 17]byte + var x13071 [1 << 17]byte + var x13072 [1 << 17]byte + var x13073 [1 << 17]byte + var x13074 [1 << 17]byte + var x13075 [1 << 17]byte + var x13076 [1 << 17]byte + var x13077 [1 << 17]byte + var x13078 [1 << 17]byte + var x13079 [1 << 17]byte + var x13080 [1 << 17]byte + var x13081 [1 << 17]byte + var x13082 [1 << 17]byte + var x13083 [1 << 17]byte + var x13084 [1 << 17]byte + var x13085 [1 << 17]byte + var x13086 [1 << 17]byte + var x13087 [1 << 17]byte + var x13088 [1 << 17]byte + var x13089 [1 << 17]byte + var x13090 [1 << 17]byte + var x13091 [1 << 17]byte + var x13092 [1 << 17]byte + var x13093 [1 << 17]byte + var x13094 [1 << 17]byte + var x13095 [1 << 17]byte + var x13096 [1 << 17]byte + var x13097 [1 << 17]byte + var x13098 [1 << 17]byte + var x13099 [1 << 17]byte + var x13100 [1 << 17]byte + var x13101 [1 << 17]byte + var x13102 [1 << 17]byte + var x13103 [1 << 17]byte + var x13104 [1 << 17]byte + var x13105 [1 << 17]byte + var x13106 [1 << 17]byte + var x13107 [1 << 17]byte + var x13108 [1 << 17]byte + var x13109 [1 << 17]byte + var x13110 [1 << 17]byte + var x13111 [1 << 17]byte + var x13112 [1 << 17]byte + var x13113 [1 << 17]byte + var x13114 [1 << 17]byte + var x13115 [1 << 17]byte + var x13116 [1 << 17]byte + var x13117 [1 << 17]byte + var x13118 [1 << 17]byte + var x13119 [1 << 17]byte + var x13120 [1 << 17]byte + var x13121 [1 << 17]byte + var x13122 [1 << 17]byte + var x13123 [1 << 17]byte + var x13124 [1 << 17]byte + var x13125 [1 << 17]byte + var x13126 [1 << 17]byte + var x13127 [1 << 17]byte + var x13128 [1 << 17]byte + var x13129 [1 << 17]byte + var x13130 [1 << 17]byte + var x13131 [1 << 17]byte + var x13132 [1 << 17]byte + var x13133 [1 << 17]byte + var x13134 [1 << 17]byte + var x13135 [1 << 17]byte + var x13136 [1 << 17]byte + var x13137 [1 << 17]byte + var x13138 [1 << 17]byte + var x13139 [1 << 17]byte + var x13140 [1 << 17]byte + var x13141 [1 << 17]byte + var x13142 [1 << 17]byte + var x13143 [1 << 17]byte + var x13144 [1 << 17]byte + var x13145 [1 << 17]byte + var x13146 [1 << 17]byte + var x13147 [1 << 17]byte + var x13148 [1 << 17]byte + var x13149 [1 << 17]byte + var x13150 [1 << 17]byte + var x13151 [1 << 17]byte + var x13152 [1 << 17]byte + var x13153 [1 << 17]byte + var x13154 [1 << 17]byte + var x13155 [1 << 17]byte + var x13156 [1 << 17]byte + var x13157 [1 << 17]byte + var x13158 [1 << 17]byte + var x13159 [1 << 17]byte + var x13160 [1 << 17]byte + var x13161 [1 << 17]byte + var x13162 [1 << 17]byte + var x13163 [1 << 17]byte + var x13164 [1 << 17]byte + var x13165 [1 << 17]byte + var x13166 [1 << 17]byte + var x13167 [1 << 17]byte + var x13168 [1 << 17]byte + var x13169 [1 << 17]byte + var x13170 [1 << 17]byte + var x13171 [1 << 17]byte + var x13172 [1 << 17]byte + var x13173 [1 << 17]byte + var x13174 [1 << 17]byte + var x13175 [1 << 17]byte + var x13176 [1 << 17]byte + var x13177 [1 << 17]byte + var x13178 [1 << 17]byte + var x13179 [1 << 17]byte + var x13180 [1 << 17]byte + var x13181 [1 << 17]byte + var x13182 [1 << 17]byte + var x13183 [1 << 17]byte + var x13184 [1 << 17]byte + var x13185 [1 << 17]byte + var x13186 [1 << 17]byte + var x13187 [1 << 17]byte + var x13188 [1 << 17]byte + var x13189 [1 << 17]byte + var x13190 [1 << 17]byte + var x13191 [1 << 17]byte + var x13192 [1 << 17]byte + var x13193 [1 << 17]byte + var x13194 [1 << 17]byte + var x13195 [1 << 17]byte + var x13196 [1 << 17]byte + var x13197 [1 << 17]byte + var x13198 [1 << 17]byte + var x13199 [1 << 17]byte + var x13200 [1 << 17]byte + var x13201 [1 << 17]byte + var x13202 [1 << 17]byte + var x13203 [1 << 17]byte + var x13204 [1 << 17]byte + var x13205 [1 << 17]byte + var x13206 [1 << 17]byte + var x13207 [1 << 17]byte + var x13208 [1 << 17]byte + var x13209 [1 << 17]byte + var x13210 [1 << 17]byte + var x13211 [1 << 17]byte + var x13212 [1 << 17]byte + var x13213 [1 << 17]byte + var x13214 [1 << 17]byte + var x13215 [1 << 17]byte + var x13216 [1 << 17]byte + var x13217 [1 << 17]byte + var x13218 [1 << 17]byte + var x13219 [1 << 17]byte + var x13220 [1 << 17]byte + var x13221 [1 << 17]byte + var x13222 [1 << 17]byte + var x13223 [1 << 17]byte + var x13224 [1 << 17]byte + var x13225 [1 << 17]byte + var x13226 [1 << 17]byte + var x13227 [1 << 17]byte + var x13228 [1 << 17]byte + var x13229 [1 << 17]byte + var x13230 [1 << 17]byte + var x13231 [1 << 17]byte + var x13232 [1 << 17]byte + var x13233 [1 << 17]byte + var x13234 [1 << 17]byte + var x13235 [1 << 17]byte + var x13236 [1 << 17]byte + var x13237 [1 << 17]byte + var x13238 [1 << 17]byte + var x13239 [1 << 17]byte + var x13240 [1 << 17]byte + var x13241 [1 << 17]byte + var x13242 [1 << 17]byte + var x13243 [1 << 17]byte + var x13244 [1 << 17]byte + var x13245 [1 << 17]byte + var x13246 [1 << 17]byte + var x13247 [1 << 17]byte + var x13248 [1 << 17]byte + var x13249 [1 << 17]byte + var x13250 [1 << 17]byte + var x13251 [1 << 17]byte + var x13252 [1 << 17]byte + var x13253 [1 << 17]byte + var x13254 [1 << 17]byte + var x13255 [1 << 17]byte + var x13256 [1 << 17]byte + var x13257 [1 << 17]byte + var x13258 [1 << 17]byte + var x13259 [1 << 17]byte + var x13260 [1 << 17]byte + var x13261 [1 << 17]byte + var x13262 [1 << 17]byte + var x13263 [1 << 17]byte + var x13264 [1 << 17]byte + var x13265 [1 << 17]byte + var x13266 [1 << 17]byte + var x13267 [1 << 17]byte + var x13268 [1 << 17]byte + var x13269 [1 << 17]byte + var x13270 [1 << 17]byte + var x13271 [1 << 17]byte + var x13272 [1 << 17]byte + var x13273 [1 << 17]byte + var x13274 [1 << 17]byte + var x13275 [1 << 17]byte + var x13276 [1 << 17]byte + var x13277 [1 << 17]byte + var x13278 [1 << 17]byte + var x13279 [1 << 17]byte + var x13280 [1 << 17]byte + var x13281 [1 << 17]byte + var x13282 [1 << 17]byte + var x13283 [1 << 17]byte + var x13284 [1 << 17]byte + var x13285 [1 << 17]byte + var x13286 [1 << 17]byte + var x13287 [1 << 17]byte + var x13288 [1 << 17]byte + var x13289 [1 << 17]byte + var x13290 [1 << 17]byte + var x13291 [1 << 17]byte + var x13292 [1 << 17]byte + var x13293 [1 << 17]byte + var x13294 [1 << 17]byte + var x13295 [1 << 17]byte + var x13296 [1 << 17]byte + var x13297 [1 << 17]byte + var x13298 [1 << 17]byte + var x13299 [1 << 17]byte + var x13300 [1 << 17]byte + var x13301 [1 << 17]byte + var x13302 [1 << 17]byte + var x13303 [1 << 17]byte + var x13304 [1 << 17]byte + var x13305 [1 << 17]byte + var x13306 [1 << 17]byte + var x13307 [1 << 17]byte + var x13308 [1 << 17]byte + var x13309 [1 << 17]byte + var x13310 [1 << 17]byte + var x13311 [1 << 17]byte + var x13312 [1 << 17]byte + var x13313 [1 << 17]byte + var x13314 [1 << 17]byte + var x13315 [1 << 17]byte + var x13316 [1 << 17]byte + var x13317 [1 << 17]byte + var x13318 [1 << 17]byte + var x13319 [1 << 17]byte + var x13320 [1 << 17]byte + var x13321 [1 << 17]byte + var x13322 [1 << 17]byte + var x13323 [1 << 17]byte + var x13324 [1 << 17]byte + var x13325 [1 << 17]byte + var x13326 [1 << 17]byte + var x13327 [1 << 17]byte + var x13328 [1 << 17]byte + var x13329 [1 << 17]byte + var x13330 [1 << 17]byte + var x13331 [1 << 17]byte + var x13332 [1 << 17]byte + var x13333 [1 << 17]byte + var x13334 [1 << 17]byte + var x13335 [1 << 17]byte + var x13336 [1 << 17]byte + var x13337 [1 << 17]byte + var x13338 [1 << 17]byte + var x13339 [1 << 17]byte + var x13340 [1 << 17]byte + var x13341 [1 << 17]byte + var x13342 [1 << 17]byte + var x13343 [1 << 17]byte + var x13344 [1 << 17]byte + var x13345 [1 << 17]byte + var x13346 [1 << 17]byte + var x13347 [1 << 17]byte + var x13348 [1 << 17]byte + var x13349 [1 << 17]byte + var x13350 [1 << 17]byte + var x13351 [1 << 17]byte + var x13352 [1 << 17]byte + var x13353 [1 << 17]byte + var x13354 [1 << 17]byte + var x13355 [1 << 17]byte + var x13356 [1 << 17]byte + var x13357 [1 << 17]byte + var x13358 [1 << 17]byte + var x13359 [1 << 17]byte + var x13360 [1 << 17]byte + var x13361 [1 << 17]byte + var x13362 [1 << 17]byte + var x13363 [1 << 17]byte + var x13364 [1 << 17]byte + var x13365 [1 << 17]byte + var x13366 [1 << 17]byte + var x13367 [1 << 17]byte + var x13368 [1 << 17]byte + var x13369 [1 << 17]byte + var x13370 [1 << 17]byte + var x13371 [1 << 17]byte + var x13372 [1 << 17]byte + var x13373 [1 << 17]byte + var x13374 [1 << 17]byte + var x13375 [1 << 17]byte + var x13376 [1 << 17]byte + var x13377 [1 << 17]byte + var x13378 [1 << 17]byte + var x13379 [1 << 17]byte + var x13380 [1 << 17]byte + var x13381 [1 << 17]byte + var x13382 [1 << 17]byte + var x13383 [1 << 17]byte + var x13384 [1 << 17]byte + var x13385 [1 << 17]byte + var x13386 [1 << 17]byte + var x13387 [1 << 17]byte + var x13388 [1 << 17]byte + var x13389 [1 << 17]byte + var x13390 [1 << 17]byte + var x13391 [1 << 17]byte + var x13392 [1 << 17]byte + var x13393 [1 << 17]byte + var x13394 [1 << 17]byte + var x13395 [1 << 17]byte + var x13396 [1 << 17]byte + var x13397 [1 << 17]byte + var x13398 [1 << 17]byte + var x13399 [1 << 17]byte + var x13400 [1 << 17]byte + var x13401 [1 << 17]byte + var x13402 [1 << 17]byte + var x13403 [1 << 17]byte + var x13404 [1 << 17]byte + var x13405 [1 << 17]byte + var x13406 [1 << 17]byte + var x13407 [1 << 17]byte + var x13408 [1 << 17]byte + var x13409 [1 << 17]byte + var x13410 [1 << 17]byte + var x13411 [1 << 17]byte + var x13412 [1 << 17]byte + var x13413 [1 << 17]byte + var x13414 [1 << 17]byte + var x13415 [1 << 17]byte + var x13416 [1 << 17]byte + var x13417 [1 << 17]byte + var x13418 [1 << 17]byte + var x13419 [1 << 17]byte + var x13420 [1 << 17]byte + var x13421 [1 << 17]byte + var x13422 [1 << 17]byte + var x13423 [1 << 17]byte + var x13424 [1 << 17]byte + var x13425 [1 << 17]byte + var x13426 [1 << 17]byte + var x13427 [1 << 17]byte + var x13428 [1 << 17]byte + var x13429 [1 << 17]byte + var x13430 [1 << 17]byte + var x13431 [1 << 17]byte + var x13432 [1 << 17]byte + var x13433 [1 << 17]byte + var x13434 [1 << 17]byte + var x13435 [1 << 17]byte + var x13436 [1 << 17]byte + var x13437 [1 << 17]byte + var x13438 [1 << 17]byte + var x13439 [1 << 17]byte + var x13440 [1 << 17]byte + var x13441 [1 << 17]byte + var x13442 [1 << 17]byte + var x13443 [1 << 17]byte + var x13444 [1 << 17]byte + var x13445 [1 << 17]byte + var x13446 [1 << 17]byte + var x13447 [1 << 17]byte + var x13448 [1 << 17]byte + var x13449 [1 << 17]byte + var x13450 [1 << 17]byte + var x13451 [1 << 17]byte + var x13452 [1 << 17]byte + var x13453 [1 << 17]byte + var x13454 [1 << 17]byte + var x13455 [1 << 17]byte + var x13456 [1 << 17]byte + var x13457 [1 << 17]byte + var x13458 [1 << 17]byte + var x13459 [1 << 17]byte + var x13460 [1 << 17]byte + var x13461 [1 << 17]byte + var x13462 [1 << 17]byte + var x13463 [1 << 17]byte + var x13464 [1 << 17]byte + var x13465 [1 << 17]byte + var x13466 [1 << 17]byte + var x13467 [1 << 17]byte + var x13468 [1 << 17]byte + var x13469 [1 << 17]byte + var x13470 [1 << 17]byte + var x13471 [1 << 17]byte + var x13472 [1 << 17]byte + var x13473 [1 << 17]byte + var x13474 [1 << 17]byte + var x13475 [1 << 17]byte + var x13476 [1 << 17]byte + var x13477 [1 << 17]byte + var x13478 [1 << 17]byte + var x13479 [1 << 17]byte + var x13480 [1 << 17]byte + var x13481 [1 << 17]byte + var x13482 [1 << 17]byte + var x13483 [1 << 17]byte + var x13484 [1 << 17]byte + var x13485 [1 << 17]byte + var x13486 [1 << 17]byte + var x13487 [1 << 17]byte + var x13488 [1 << 17]byte + var x13489 [1 << 17]byte + var x13490 [1 << 17]byte + var x13491 [1 << 17]byte + var x13492 [1 << 17]byte + var x13493 [1 << 17]byte + var x13494 [1 << 17]byte + var x13495 [1 << 17]byte + var x13496 [1 << 17]byte + var x13497 [1 << 17]byte + var x13498 [1 << 17]byte + var x13499 [1 << 17]byte + var x13500 [1 << 17]byte + var x13501 [1 << 17]byte + var x13502 [1 << 17]byte + var x13503 [1 << 17]byte + var x13504 [1 << 17]byte + var x13505 [1 << 17]byte + var x13506 [1 << 17]byte + var x13507 [1 << 17]byte + var x13508 [1 << 17]byte + var x13509 [1 << 17]byte + var x13510 [1 << 17]byte + var x13511 [1 << 17]byte + var x13512 [1 << 17]byte + var x13513 [1 << 17]byte + var x13514 [1 << 17]byte + var x13515 [1 << 17]byte + var x13516 [1 << 17]byte + var x13517 [1 << 17]byte + var x13518 [1 << 17]byte + var x13519 [1 << 17]byte + var x13520 [1 << 17]byte + var x13521 [1 << 17]byte + var x13522 [1 << 17]byte + var x13523 [1 << 17]byte + var x13524 [1 << 17]byte + var x13525 [1 << 17]byte + var x13526 [1 << 17]byte + var x13527 [1 << 17]byte + var x13528 [1 << 17]byte + var x13529 [1 << 17]byte + var x13530 [1 << 17]byte + var x13531 [1 << 17]byte + var x13532 [1 << 17]byte + var x13533 [1 << 17]byte + var x13534 [1 << 17]byte + var x13535 [1 << 17]byte + var x13536 [1 << 17]byte + var x13537 [1 << 17]byte + var x13538 [1 << 17]byte + var x13539 [1 << 17]byte + var x13540 [1 << 17]byte + var x13541 [1 << 17]byte + var x13542 [1 << 17]byte + var x13543 [1 << 17]byte + var x13544 [1 << 17]byte + var x13545 [1 << 17]byte + var x13546 [1 << 17]byte + var x13547 [1 << 17]byte + var x13548 [1 << 17]byte + var x13549 [1 << 17]byte + var x13550 [1 << 17]byte + var x13551 [1 << 17]byte + var x13552 [1 << 17]byte + var x13553 [1 << 17]byte + var x13554 [1 << 17]byte + var x13555 [1 << 17]byte + var x13556 [1 << 17]byte + var x13557 [1 << 17]byte + var x13558 [1 << 17]byte + var x13559 [1 << 17]byte + var x13560 [1 << 17]byte + var x13561 [1 << 17]byte + var x13562 [1 << 17]byte + var x13563 [1 << 17]byte + var x13564 [1 << 17]byte + var x13565 [1 << 17]byte + var x13566 [1 << 17]byte + var x13567 [1 << 17]byte + var x13568 [1 << 17]byte + var x13569 [1 << 17]byte + var x13570 [1 << 17]byte + var x13571 [1 << 17]byte + var x13572 [1 << 17]byte + var x13573 [1 << 17]byte + var x13574 [1 << 17]byte + var x13575 [1 << 17]byte + var x13576 [1 << 17]byte + var x13577 [1 << 17]byte + var x13578 [1 << 17]byte + var x13579 [1 << 17]byte + var x13580 [1 << 17]byte + var x13581 [1 << 17]byte + var x13582 [1 << 17]byte + var x13583 [1 << 17]byte + var x13584 [1 << 17]byte + var x13585 [1 << 17]byte + var x13586 [1 << 17]byte + var x13587 [1 << 17]byte + var x13588 [1 << 17]byte + var x13589 [1 << 17]byte + var x13590 [1 << 17]byte + var x13591 [1 << 17]byte + var x13592 [1 << 17]byte + var x13593 [1 << 17]byte + var x13594 [1 << 17]byte + var x13595 [1 << 17]byte + var x13596 [1 << 17]byte + var x13597 [1 << 17]byte + var x13598 [1 << 17]byte + var x13599 [1 << 17]byte + var x13600 [1 << 17]byte + var x13601 [1 << 17]byte + var x13602 [1 << 17]byte + var x13603 [1 << 17]byte + var x13604 [1 << 17]byte + var x13605 [1 << 17]byte + var x13606 [1 << 17]byte + var x13607 [1 << 17]byte + var x13608 [1 << 17]byte + var x13609 [1 << 17]byte + var x13610 [1 << 17]byte + var x13611 [1 << 17]byte + var x13612 [1 << 17]byte + var x13613 [1 << 17]byte + var x13614 [1 << 17]byte + var x13615 [1 << 17]byte + var x13616 [1 << 17]byte + var x13617 [1 << 17]byte + var x13618 [1 << 17]byte + var x13619 [1 << 17]byte + var x13620 [1 << 17]byte + var x13621 [1 << 17]byte + var x13622 [1 << 17]byte + var x13623 [1 << 17]byte + var x13624 [1 << 17]byte + var x13625 [1 << 17]byte + var x13626 [1 << 17]byte + var x13627 [1 << 17]byte + var x13628 [1 << 17]byte + var x13629 [1 << 17]byte + var x13630 [1 << 17]byte + var x13631 [1 << 17]byte + var x13632 [1 << 17]byte + var x13633 [1 << 17]byte + var x13634 [1 << 17]byte + var x13635 [1 << 17]byte + var x13636 [1 << 17]byte + var x13637 [1 << 17]byte + var x13638 [1 << 17]byte + var x13639 [1 << 17]byte + var x13640 [1 << 17]byte + var x13641 [1 << 17]byte + var x13642 [1 << 17]byte + var x13643 [1 << 17]byte + var x13644 [1 << 17]byte + var x13645 [1 << 17]byte + var x13646 [1 << 17]byte + var x13647 [1 << 17]byte + var x13648 [1 << 17]byte + var x13649 [1 << 17]byte + var x13650 [1 << 17]byte + var x13651 [1 << 17]byte + var x13652 [1 << 17]byte + var x13653 [1 << 17]byte + var x13654 [1 << 17]byte + var x13655 [1 << 17]byte + var x13656 [1 << 17]byte + var x13657 [1 << 17]byte + var x13658 [1 << 17]byte + var x13659 [1 << 17]byte + var x13660 [1 << 17]byte + var x13661 [1 << 17]byte + var x13662 [1 << 17]byte + var x13663 [1 << 17]byte + var x13664 [1 << 17]byte + var x13665 [1 << 17]byte + var x13666 [1 << 17]byte + var x13667 [1 << 17]byte + var x13668 [1 << 17]byte + var x13669 [1 << 17]byte + var x13670 [1 << 17]byte + var x13671 [1 << 17]byte + var x13672 [1 << 17]byte + var x13673 [1 << 17]byte + var x13674 [1 << 17]byte + var x13675 [1 << 17]byte + var x13676 [1 << 17]byte + var x13677 [1 << 17]byte + var x13678 [1 << 17]byte + var x13679 [1 << 17]byte + var x13680 [1 << 17]byte + var x13681 [1 << 17]byte + var x13682 [1 << 17]byte + var x13683 [1 << 17]byte + var x13684 [1 << 17]byte + var x13685 [1 << 17]byte + var x13686 [1 << 17]byte + var x13687 [1 << 17]byte + var x13688 [1 << 17]byte + var x13689 [1 << 17]byte + var x13690 [1 << 17]byte + var x13691 [1 << 17]byte + var x13692 [1 << 17]byte + var x13693 [1 << 17]byte + var x13694 [1 << 17]byte + var x13695 [1 << 17]byte + var x13696 [1 << 17]byte + var x13697 [1 << 17]byte + var x13698 [1 << 17]byte + var x13699 [1 << 17]byte + var x13700 [1 << 17]byte + var x13701 [1 << 17]byte + var x13702 [1 << 17]byte + var x13703 [1 << 17]byte + var x13704 [1 << 17]byte + var x13705 [1 << 17]byte + var x13706 [1 << 17]byte + var x13707 [1 << 17]byte + var x13708 [1 << 17]byte + var x13709 [1 << 17]byte + var x13710 [1 << 17]byte + var x13711 [1 << 17]byte + var x13712 [1 << 17]byte + var x13713 [1 << 17]byte + var x13714 [1 << 17]byte + var x13715 [1 << 17]byte + var x13716 [1 << 17]byte + var x13717 [1 << 17]byte + var x13718 [1 << 17]byte + var x13719 [1 << 17]byte + var x13720 [1 << 17]byte + var x13721 [1 << 17]byte + var x13722 [1 << 17]byte + var x13723 [1 << 17]byte + var x13724 [1 << 17]byte + var x13725 [1 << 17]byte + var x13726 [1 << 17]byte + var x13727 [1 << 17]byte + var x13728 [1 << 17]byte + var x13729 [1 << 17]byte + var x13730 [1 << 17]byte + var x13731 [1 << 17]byte + var x13732 [1 << 17]byte + var x13733 [1 << 17]byte + var x13734 [1 << 17]byte + var x13735 [1 << 17]byte + var x13736 [1 << 17]byte + var x13737 [1 << 17]byte + var x13738 [1 << 17]byte + var x13739 [1 << 17]byte + var x13740 [1 << 17]byte + var x13741 [1 << 17]byte + var x13742 [1 << 17]byte + var x13743 [1 << 17]byte + var x13744 [1 << 17]byte + var x13745 [1 << 17]byte + var x13746 [1 << 17]byte + var x13747 [1 << 17]byte + var x13748 [1 << 17]byte + var x13749 [1 << 17]byte + var x13750 [1 << 17]byte + var x13751 [1 << 17]byte + var x13752 [1 << 17]byte + var x13753 [1 << 17]byte + var x13754 [1 << 17]byte + var x13755 [1 << 17]byte + var x13756 [1 << 17]byte + var x13757 [1 << 17]byte + var x13758 [1 << 17]byte + var x13759 [1 << 17]byte + var x13760 [1 << 17]byte + var x13761 [1 << 17]byte + var x13762 [1 << 17]byte + var x13763 [1 << 17]byte + var x13764 [1 << 17]byte + var x13765 [1 << 17]byte + var x13766 [1 << 17]byte + var x13767 [1 << 17]byte + var x13768 [1 << 17]byte + var x13769 [1 << 17]byte + var x13770 [1 << 17]byte + var x13771 [1 << 17]byte + var x13772 [1 << 17]byte + var x13773 [1 << 17]byte + var x13774 [1 << 17]byte + var x13775 [1 << 17]byte + var x13776 [1 << 17]byte + var x13777 [1 << 17]byte + var x13778 [1 << 17]byte + var x13779 [1 << 17]byte + var x13780 [1 << 17]byte + var x13781 [1 << 17]byte + var x13782 [1 << 17]byte + var x13783 [1 << 17]byte + var x13784 [1 << 17]byte + var x13785 [1 << 17]byte + var x13786 [1 << 17]byte + var x13787 [1 << 17]byte + var x13788 [1 << 17]byte + var x13789 [1 << 17]byte + var x13790 [1 << 17]byte + var x13791 [1 << 17]byte + var x13792 [1 << 17]byte + var x13793 [1 << 17]byte + var x13794 [1 << 17]byte + var x13795 [1 << 17]byte + var x13796 [1 << 17]byte + var x13797 [1 << 17]byte + var x13798 [1 << 17]byte + var x13799 [1 << 17]byte + var x13800 [1 << 17]byte + var x13801 [1 << 17]byte + var x13802 [1 << 17]byte + var x13803 [1 << 17]byte + var x13804 [1 << 17]byte + var x13805 [1 << 17]byte + var x13806 [1 << 17]byte + var x13807 [1 << 17]byte + var x13808 [1 << 17]byte + var x13809 [1 << 17]byte + var x13810 [1 << 17]byte + var x13811 [1 << 17]byte + var x13812 [1 << 17]byte + var x13813 [1 << 17]byte + var x13814 [1 << 17]byte + var x13815 [1 << 17]byte + var x13816 [1 << 17]byte + var x13817 [1 << 17]byte + var x13818 [1 << 17]byte + var x13819 [1 << 17]byte + var x13820 [1 << 17]byte + var x13821 [1 << 17]byte + var x13822 [1 << 17]byte + var x13823 [1 << 17]byte + var x13824 [1 << 17]byte + var x13825 [1 << 17]byte + var x13826 [1 << 17]byte + var x13827 [1 << 17]byte + var x13828 [1 << 17]byte + var x13829 [1 << 17]byte + var x13830 [1 << 17]byte + var x13831 [1 << 17]byte + var x13832 [1 << 17]byte + var x13833 [1 << 17]byte + var x13834 [1 << 17]byte + var x13835 [1 << 17]byte + var x13836 [1 << 17]byte + var x13837 [1 << 17]byte + var x13838 [1 << 17]byte + var x13839 [1 << 17]byte + var x13840 [1 << 17]byte + var x13841 [1 << 17]byte + var x13842 [1 << 17]byte + var x13843 [1 << 17]byte + var x13844 [1 << 17]byte + var x13845 [1 << 17]byte + var x13846 [1 << 17]byte + var x13847 [1 << 17]byte + var x13848 [1 << 17]byte + var x13849 [1 << 17]byte + var x13850 [1 << 17]byte + var x13851 [1 << 17]byte + var x13852 [1 << 17]byte + var x13853 [1 << 17]byte + var x13854 [1 << 17]byte + var x13855 [1 << 17]byte + var x13856 [1 << 17]byte + var x13857 [1 << 17]byte + var x13858 [1 << 17]byte + var x13859 [1 << 17]byte + var x13860 [1 << 17]byte + var x13861 [1 << 17]byte + var x13862 [1 << 17]byte + var x13863 [1 << 17]byte + var x13864 [1 << 17]byte + var x13865 [1 << 17]byte + var x13866 [1 << 17]byte + var x13867 [1 << 17]byte + var x13868 [1 << 17]byte + var x13869 [1 << 17]byte + var x13870 [1 << 17]byte + var x13871 [1 << 17]byte + var x13872 [1 << 17]byte + var x13873 [1 << 17]byte + var x13874 [1 << 17]byte + var x13875 [1 << 17]byte + var x13876 [1 << 17]byte + var x13877 [1 << 17]byte + var x13878 [1 << 17]byte + var x13879 [1 << 17]byte + var x13880 [1 << 17]byte + var x13881 [1 << 17]byte + var x13882 [1 << 17]byte + var x13883 [1 << 17]byte + var x13884 [1 << 17]byte + var x13885 [1 << 17]byte + var x13886 [1 << 17]byte + var x13887 [1 << 17]byte + var x13888 [1 << 17]byte + var x13889 [1 << 17]byte + var x13890 [1 << 17]byte + var x13891 [1 << 17]byte + var x13892 [1 << 17]byte + var x13893 [1 << 17]byte + var x13894 [1 << 17]byte + var x13895 [1 << 17]byte + var x13896 [1 << 17]byte + var x13897 [1 << 17]byte + var x13898 [1 << 17]byte + var x13899 [1 << 17]byte + var x13900 [1 << 17]byte + var x13901 [1 << 17]byte + var x13902 [1 << 17]byte + var x13903 [1 << 17]byte + var x13904 [1 << 17]byte + var x13905 [1 << 17]byte + var x13906 [1 << 17]byte + var x13907 [1 << 17]byte + var x13908 [1 << 17]byte + var x13909 [1 << 17]byte + var x13910 [1 << 17]byte + var x13911 [1 << 17]byte + var x13912 [1 << 17]byte + var x13913 [1 << 17]byte + var x13914 [1 << 17]byte + var x13915 [1 << 17]byte + var x13916 [1 << 17]byte + var x13917 [1 << 17]byte + var x13918 [1 << 17]byte + var x13919 [1 << 17]byte + var x13920 [1 << 17]byte + var x13921 [1 << 17]byte + var x13922 [1 << 17]byte + var x13923 [1 << 17]byte + var x13924 [1 << 17]byte + var x13925 [1 << 17]byte + var x13926 [1 << 17]byte + var x13927 [1 << 17]byte + var x13928 [1 << 17]byte + var x13929 [1 << 17]byte + var x13930 [1 << 17]byte + var x13931 [1 << 17]byte + var x13932 [1 << 17]byte + var x13933 [1 << 17]byte + var x13934 [1 << 17]byte + var x13935 [1 << 17]byte + var x13936 [1 << 17]byte + var x13937 [1 << 17]byte + var x13938 [1 << 17]byte + var x13939 [1 << 17]byte + var x13940 [1 << 17]byte + var x13941 [1 << 17]byte + var x13942 [1 << 17]byte + var x13943 [1 << 17]byte + var x13944 [1 << 17]byte + var x13945 [1 << 17]byte + var x13946 [1 << 17]byte + var x13947 [1 << 17]byte + var x13948 [1 << 17]byte + var x13949 [1 << 17]byte + var x13950 [1 << 17]byte + var x13951 [1 << 17]byte + var x13952 [1 << 17]byte + var x13953 [1 << 17]byte + var x13954 [1 << 17]byte + var x13955 [1 << 17]byte + var x13956 [1 << 17]byte + var x13957 [1 << 17]byte + var x13958 [1 << 17]byte + var x13959 [1 << 17]byte + var x13960 [1 << 17]byte + var x13961 [1 << 17]byte + var x13962 [1 << 17]byte + var x13963 [1 << 17]byte + var x13964 [1 << 17]byte + var x13965 [1 << 17]byte + var x13966 [1 << 17]byte + var x13967 [1 << 17]byte + var x13968 [1 << 17]byte + var x13969 [1 << 17]byte + var x13970 [1 << 17]byte + var x13971 [1 << 17]byte + var x13972 [1 << 17]byte + var x13973 [1 << 17]byte + var x13974 [1 << 17]byte + var x13975 [1 << 17]byte + var x13976 [1 << 17]byte + var x13977 [1 << 17]byte + var x13978 [1 << 17]byte + var x13979 [1 << 17]byte + var x13980 [1 << 17]byte + var x13981 [1 << 17]byte + var x13982 [1 << 17]byte + var x13983 [1 << 17]byte + var x13984 [1 << 17]byte + var x13985 [1 << 17]byte + var x13986 [1 << 17]byte + var x13987 [1 << 17]byte + var x13988 [1 << 17]byte + var x13989 [1 << 17]byte + var x13990 [1 << 17]byte + var x13991 [1 << 17]byte + var x13992 [1 << 17]byte + var x13993 [1 << 17]byte + var x13994 [1 << 17]byte + var x13995 [1 << 17]byte + var x13996 [1 << 17]byte + var x13997 [1 << 17]byte + var x13998 [1 << 17]byte + var x13999 [1 << 17]byte + var x14000 [1 << 17]byte + var x14001 [1 << 17]byte + var x14002 [1 << 17]byte + var x14003 [1 << 17]byte + var x14004 [1 << 17]byte + var x14005 [1 << 17]byte + var x14006 [1 << 17]byte + var x14007 [1 << 17]byte + var x14008 [1 << 17]byte + var x14009 [1 << 17]byte + var x14010 [1 << 17]byte + var x14011 [1 << 17]byte + var x14012 [1 << 17]byte + var x14013 [1 << 17]byte + var x14014 [1 << 17]byte + var x14015 [1 << 17]byte + var x14016 [1 << 17]byte + var x14017 [1 << 17]byte + var x14018 [1 << 17]byte + var x14019 [1 << 17]byte + var x14020 [1 << 17]byte + var x14021 [1 << 17]byte + var x14022 [1 << 17]byte + var x14023 [1 << 17]byte + var x14024 [1 << 17]byte + var x14025 [1 << 17]byte + var x14026 [1 << 17]byte + var x14027 [1 << 17]byte + var x14028 [1 << 17]byte + var x14029 [1 << 17]byte + var x14030 [1 << 17]byte + var x14031 [1 << 17]byte + var x14032 [1 << 17]byte + var x14033 [1 << 17]byte + var x14034 [1 << 17]byte + var x14035 [1 << 17]byte + var x14036 [1 << 17]byte + var x14037 [1 << 17]byte + var x14038 [1 << 17]byte + var x14039 [1 << 17]byte + var x14040 [1 << 17]byte + var x14041 [1 << 17]byte + var x14042 [1 << 17]byte + var x14043 [1 << 17]byte + var x14044 [1 << 17]byte + var x14045 [1 << 17]byte + var x14046 [1 << 17]byte + var x14047 [1 << 17]byte + var x14048 [1 << 17]byte + var x14049 [1 << 17]byte + var x14050 [1 << 17]byte + var x14051 [1 << 17]byte + var x14052 [1 << 17]byte + var x14053 [1 << 17]byte + var x14054 [1 << 17]byte + var x14055 [1 << 17]byte + var x14056 [1 << 17]byte + var x14057 [1 << 17]byte + var x14058 [1 << 17]byte + var x14059 [1 << 17]byte + var x14060 [1 << 17]byte + var x14061 [1 << 17]byte + var x14062 [1 << 17]byte + var x14063 [1 << 17]byte + var x14064 [1 << 17]byte + var x14065 [1 << 17]byte + var x14066 [1 << 17]byte + var x14067 [1 << 17]byte + var x14068 [1 << 17]byte + var x14069 [1 << 17]byte + var x14070 [1 << 17]byte + var x14071 [1 << 17]byte + var x14072 [1 << 17]byte + var x14073 [1 << 17]byte + var x14074 [1 << 17]byte + var x14075 [1 << 17]byte + var x14076 [1 << 17]byte + var x14077 [1 << 17]byte + var x14078 [1 << 17]byte + var x14079 [1 << 17]byte + var x14080 [1 << 17]byte + var x14081 [1 << 17]byte + var x14082 [1 << 17]byte + var x14083 [1 << 17]byte + var x14084 [1 << 17]byte + var x14085 [1 << 17]byte + var x14086 [1 << 17]byte + var x14087 [1 << 17]byte + var x14088 [1 << 17]byte + var x14089 [1 << 17]byte + var x14090 [1 << 17]byte + var x14091 [1 << 17]byte + var x14092 [1 << 17]byte + var x14093 [1 << 17]byte + var x14094 [1 << 17]byte + var x14095 [1 << 17]byte + var x14096 [1 << 17]byte + var x14097 [1 << 17]byte + var x14098 [1 << 17]byte + var x14099 [1 << 17]byte + var x14100 [1 << 17]byte + var x14101 [1 << 17]byte + var x14102 [1 << 17]byte + var x14103 [1 << 17]byte + var x14104 [1 << 17]byte + var x14105 [1 << 17]byte + var x14106 [1 << 17]byte + var x14107 [1 << 17]byte + var x14108 [1 << 17]byte + var x14109 [1 << 17]byte + var x14110 [1 << 17]byte + var x14111 [1 << 17]byte + var x14112 [1 << 17]byte + var x14113 [1 << 17]byte + var x14114 [1 << 17]byte + var x14115 [1 << 17]byte + var x14116 [1 << 17]byte + var x14117 [1 << 17]byte + var x14118 [1 << 17]byte + var x14119 [1 << 17]byte + var x14120 [1 << 17]byte + var x14121 [1 << 17]byte + var x14122 [1 << 17]byte + var x14123 [1 << 17]byte + var x14124 [1 << 17]byte + var x14125 [1 << 17]byte + var x14126 [1 << 17]byte + var x14127 [1 << 17]byte + var x14128 [1 << 17]byte + var x14129 [1 << 17]byte + var x14130 [1 << 17]byte + var x14131 [1 << 17]byte + var x14132 [1 << 17]byte + var x14133 [1 << 17]byte + var x14134 [1 << 17]byte + var x14135 [1 << 17]byte + var x14136 [1 << 17]byte + var x14137 [1 << 17]byte + var x14138 [1 << 17]byte + var x14139 [1 << 17]byte + var x14140 [1 << 17]byte + var x14141 [1 << 17]byte + var x14142 [1 << 17]byte + var x14143 [1 << 17]byte + var x14144 [1 << 17]byte + var x14145 [1 << 17]byte + var x14146 [1 << 17]byte + var x14147 [1 << 17]byte + var x14148 [1 << 17]byte + var x14149 [1 << 17]byte + var x14150 [1 << 17]byte + var x14151 [1 << 17]byte + var x14152 [1 << 17]byte + var x14153 [1 << 17]byte + var x14154 [1 << 17]byte + var x14155 [1 << 17]byte + var x14156 [1 << 17]byte + var x14157 [1 << 17]byte + var x14158 [1 << 17]byte + var x14159 [1 << 17]byte + var x14160 [1 << 17]byte + var x14161 [1 << 17]byte + var x14162 [1 << 17]byte + var x14163 [1 << 17]byte + var x14164 [1 << 17]byte + var x14165 [1 << 17]byte + var x14166 [1 << 17]byte + var x14167 [1 << 17]byte + var x14168 [1 << 17]byte + var x14169 [1 << 17]byte + var x14170 [1 << 17]byte + var x14171 [1 << 17]byte + var x14172 [1 << 17]byte + var x14173 [1 << 17]byte + var x14174 [1 << 17]byte + var x14175 [1 << 17]byte + var x14176 [1 << 17]byte + var x14177 [1 << 17]byte + var x14178 [1 << 17]byte + var x14179 [1 << 17]byte + var x14180 [1 << 17]byte + var x14181 [1 << 17]byte + var x14182 [1 << 17]byte + var x14183 [1 << 17]byte + var x14184 [1 << 17]byte + var x14185 [1 << 17]byte + var x14186 [1 << 17]byte + var x14187 [1 << 17]byte + var x14188 [1 << 17]byte + var x14189 [1 << 17]byte + var x14190 [1 << 17]byte + var x14191 [1 << 17]byte + var x14192 [1 << 17]byte + var x14193 [1 << 17]byte + var x14194 [1 << 17]byte + var x14195 [1 << 17]byte + var x14196 [1 << 17]byte + var x14197 [1 << 17]byte + var x14198 [1 << 17]byte + var x14199 [1 << 17]byte + var x14200 [1 << 17]byte + var x14201 [1 << 17]byte + var x14202 [1 << 17]byte + var x14203 [1 << 17]byte + var x14204 [1 << 17]byte + var x14205 [1 << 17]byte + var x14206 [1 << 17]byte + var x14207 [1 << 17]byte + var x14208 [1 << 17]byte + var x14209 [1 << 17]byte + var x14210 [1 << 17]byte + var x14211 [1 << 17]byte + var x14212 [1 << 17]byte + var x14213 [1 << 17]byte + var x14214 [1 << 17]byte + var x14215 [1 << 17]byte + var x14216 [1 << 17]byte + var x14217 [1 << 17]byte + var x14218 [1 << 17]byte + var x14219 [1 << 17]byte + var x14220 [1 << 17]byte + var x14221 [1 << 17]byte + var x14222 [1 << 17]byte + var x14223 [1 << 17]byte + var x14224 [1 << 17]byte + var x14225 [1 << 17]byte + var x14226 [1 << 17]byte + var x14227 [1 << 17]byte + var x14228 [1 << 17]byte + var x14229 [1 << 17]byte + var x14230 [1 << 17]byte + var x14231 [1 << 17]byte + var x14232 [1 << 17]byte + var x14233 [1 << 17]byte + var x14234 [1 << 17]byte + var x14235 [1 << 17]byte + var x14236 [1 << 17]byte + var x14237 [1 << 17]byte + var x14238 [1 << 17]byte + var x14239 [1 << 17]byte + var x14240 [1 << 17]byte + var x14241 [1 << 17]byte + var x14242 [1 << 17]byte + var x14243 [1 << 17]byte + var x14244 [1 << 17]byte + var x14245 [1 << 17]byte + var x14246 [1 << 17]byte + var x14247 [1 << 17]byte + var x14248 [1 << 17]byte + var x14249 [1 << 17]byte + var x14250 [1 << 17]byte + var x14251 [1 << 17]byte + var x14252 [1 << 17]byte + var x14253 [1 << 17]byte + var x14254 [1 << 17]byte + var x14255 [1 << 17]byte + var x14256 [1 << 17]byte + var x14257 [1 << 17]byte + var x14258 [1 << 17]byte + var x14259 [1 << 17]byte + var x14260 [1 << 17]byte + var x14261 [1 << 17]byte + var x14262 [1 << 17]byte + var x14263 [1 << 17]byte + var x14264 [1 << 17]byte + var x14265 [1 << 17]byte + var x14266 [1 << 17]byte + var x14267 [1 << 17]byte + var x14268 [1 << 17]byte + var x14269 [1 << 17]byte + var x14270 [1 << 17]byte + var x14271 [1 << 17]byte + var x14272 [1 << 17]byte + var x14273 [1 << 17]byte + var x14274 [1 << 17]byte + var x14275 [1 << 17]byte + var x14276 [1 << 17]byte + var x14277 [1 << 17]byte + var x14278 [1 << 17]byte + var x14279 [1 << 17]byte + var x14280 [1 << 17]byte + var x14281 [1 << 17]byte + var x14282 [1 << 17]byte + var x14283 [1 << 17]byte + var x14284 [1 << 17]byte + var x14285 [1 << 17]byte + var x14286 [1 << 17]byte + var x14287 [1 << 17]byte + var x14288 [1 << 17]byte + var x14289 [1 << 17]byte + var x14290 [1 << 17]byte + var x14291 [1 << 17]byte + var x14292 [1 << 17]byte + var x14293 [1 << 17]byte + var x14294 [1 << 17]byte + var x14295 [1 << 17]byte + var x14296 [1 << 17]byte + var x14297 [1 << 17]byte + var x14298 [1 << 17]byte + var x14299 [1 << 17]byte + var x14300 [1 << 17]byte + var x14301 [1 << 17]byte + var x14302 [1 << 17]byte + var x14303 [1 << 17]byte + var x14304 [1 << 17]byte + var x14305 [1 << 17]byte + var x14306 [1 << 17]byte + var x14307 [1 << 17]byte + var x14308 [1 << 17]byte + var x14309 [1 << 17]byte + var x14310 [1 << 17]byte + var x14311 [1 << 17]byte + var x14312 [1 << 17]byte + var x14313 [1 << 17]byte + var x14314 [1 << 17]byte + var x14315 [1 << 17]byte + var x14316 [1 << 17]byte + var x14317 [1 << 17]byte + var x14318 [1 << 17]byte + var x14319 [1 << 17]byte + var x14320 [1 << 17]byte + var x14321 [1 << 17]byte + var x14322 [1 << 17]byte + var x14323 [1 << 17]byte + var x14324 [1 << 17]byte + var x14325 [1 << 17]byte + var x14326 [1 << 17]byte + var x14327 [1 << 17]byte + var x14328 [1 << 17]byte + var x14329 [1 << 17]byte + var x14330 [1 << 17]byte + var x14331 [1 << 17]byte + var x14332 [1 << 17]byte + var x14333 [1 << 17]byte + var x14334 [1 << 17]byte + var x14335 [1 << 17]byte + var x14336 [1 << 17]byte + var x14337 [1 << 17]byte + var x14338 [1 << 17]byte + var x14339 [1 << 17]byte + var x14340 [1 << 17]byte + var x14341 [1 << 17]byte + var x14342 [1 << 17]byte + var x14343 [1 << 17]byte + var x14344 [1 << 17]byte + var x14345 [1 << 17]byte + var x14346 [1 << 17]byte + var x14347 [1 << 17]byte + var x14348 [1 << 17]byte + var x14349 [1 << 17]byte + var x14350 [1 << 17]byte + var x14351 [1 << 17]byte + var x14352 [1 << 17]byte + var x14353 [1 << 17]byte + var x14354 [1 << 17]byte + var x14355 [1 << 17]byte + var x14356 [1 << 17]byte + var x14357 [1 << 17]byte + var x14358 [1 << 17]byte + var x14359 [1 << 17]byte + var x14360 [1 << 17]byte + var x14361 [1 << 17]byte + var x14362 [1 << 17]byte + var x14363 [1 << 17]byte + var x14364 [1 << 17]byte + var x14365 [1 << 17]byte + var x14366 [1 << 17]byte + var x14367 [1 << 17]byte + var x14368 [1 << 17]byte + var x14369 [1 << 17]byte + var x14370 [1 << 17]byte + var x14371 [1 << 17]byte + var x14372 [1 << 17]byte + var x14373 [1 << 17]byte + var x14374 [1 << 17]byte + var x14375 [1 << 17]byte + var x14376 [1 << 17]byte + var x14377 [1 << 17]byte + var x14378 [1 << 17]byte + var x14379 [1 << 17]byte + var x14380 [1 << 17]byte + var x14381 [1 << 17]byte + var x14382 [1 << 17]byte + var x14383 [1 << 17]byte + var x14384 [1 << 17]byte + var x14385 [1 << 17]byte + var x14386 [1 << 17]byte + var x14387 [1 << 17]byte + var x14388 [1 << 17]byte + var x14389 [1 << 17]byte + var x14390 [1 << 17]byte + var x14391 [1 << 17]byte + var x14392 [1 << 17]byte + var x14393 [1 << 17]byte + var x14394 [1 << 17]byte + var x14395 [1 << 17]byte + var x14396 [1 << 17]byte + var x14397 [1 << 17]byte + var x14398 [1 << 17]byte + var x14399 [1 << 17]byte + var x14400 [1 << 17]byte + var x14401 [1 << 17]byte + var x14402 [1 << 17]byte + var x14403 [1 << 17]byte + var x14404 [1 << 17]byte + var x14405 [1 << 17]byte + var x14406 [1 << 17]byte + var x14407 [1 << 17]byte + var x14408 [1 << 17]byte + var x14409 [1 << 17]byte + var x14410 [1 << 17]byte + var x14411 [1 << 17]byte + var x14412 [1 << 17]byte + var x14413 [1 << 17]byte + var x14414 [1 << 17]byte + var x14415 [1 << 17]byte + var x14416 [1 << 17]byte + var x14417 [1 << 17]byte + var x14418 [1 << 17]byte + var x14419 [1 << 17]byte + var x14420 [1 << 17]byte + var x14421 [1 << 17]byte + var x14422 [1 << 17]byte + var x14423 [1 << 17]byte + var x14424 [1 << 17]byte + var x14425 [1 << 17]byte + var x14426 [1 << 17]byte + var x14427 [1 << 17]byte + var x14428 [1 << 17]byte + var x14429 [1 << 17]byte + var x14430 [1 << 17]byte + var x14431 [1 << 17]byte + var x14432 [1 << 17]byte + var x14433 [1 << 17]byte + var x14434 [1 << 17]byte + var x14435 [1 << 17]byte + var x14436 [1 << 17]byte + var x14437 [1 << 17]byte + var x14438 [1 << 17]byte + var x14439 [1 << 17]byte + var x14440 [1 << 17]byte + var x14441 [1 << 17]byte + var x14442 [1 << 17]byte + var x14443 [1 << 17]byte + var x14444 [1 << 17]byte + var x14445 [1 << 17]byte + var x14446 [1 << 17]byte + var x14447 [1 << 17]byte + var x14448 [1 << 17]byte + var x14449 [1 << 17]byte + var x14450 [1 << 17]byte + var x14451 [1 << 17]byte + var x14452 [1 << 17]byte + var x14453 [1 << 17]byte + var x14454 [1 << 17]byte + var x14455 [1 << 17]byte + var x14456 [1 << 17]byte + var x14457 [1 << 17]byte + var x14458 [1 << 17]byte + var x14459 [1 << 17]byte + var x14460 [1 << 17]byte + var x14461 [1 << 17]byte + var x14462 [1 << 17]byte + var x14463 [1 << 17]byte + var x14464 [1 << 17]byte + var x14465 [1 << 17]byte + var x14466 [1 << 17]byte + var x14467 [1 << 17]byte + var x14468 [1 << 17]byte + var x14469 [1 << 17]byte + var x14470 [1 << 17]byte + var x14471 [1 << 17]byte + var x14472 [1 << 17]byte + var x14473 [1 << 17]byte + var x14474 [1 << 17]byte + var x14475 [1 << 17]byte + var x14476 [1 << 17]byte + var x14477 [1 << 17]byte + var x14478 [1 << 17]byte + var x14479 [1 << 17]byte + var x14480 [1 << 17]byte + var x14481 [1 << 17]byte + var x14482 [1 << 17]byte + var x14483 [1 << 17]byte + var x14484 [1 << 17]byte + var x14485 [1 << 17]byte + var x14486 [1 << 17]byte + var x14487 [1 << 17]byte + var x14488 [1 << 17]byte + var x14489 [1 << 17]byte + var x14490 [1 << 17]byte + var x14491 [1 << 17]byte + var x14492 [1 << 17]byte + var x14493 [1 << 17]byte + var x14494 [1 << 17]byte + var x14495 [1 << 17]byte + var x14496 [1 << 17]byte + var x14497 [1 << 17]byte + var x14498 [1 << 17]byte + var x14499 [1 << 17]byte + var x14500 [1 << 17]byte + var x14501 [1 << 17]byte + var x14502 [1 << 17]byte + var x14503 [1 << 17]byte + var x14504 [1 << 17]byte + var x14505 [1 << 17]byte + var x14506 [1 << 17]byte + var x14507 [1 << 17]byte + var x14508 [1 << 17]byte + var x14509 [1 << 17]byte + var x14510 [1 << 17]byte + var x14511 [1 << 17]byte + var x14512 [1 << 17]byte + var x14513 [1 << 17]byte + var x14514 [1 << 17]byte + var x14515 [1 << 17]byte + var x14516 [1 << 17]byte + var x14517 [1 << 17]byte + var x14518 [1 << 17]byte + var x14519 [1 << 17]byte + var x14520 [1 << 17]byte + var x14521 [1 << 17]byte + var x14522 [1 << 17]byte + var x14523 [1 << 17]byte + var x14524 [1 << 17]byte + var x14525 [1 << 17]byte + var x14526 [1 << 17]byte + var x14527 [1 << 17]byte + var x14528 [1 << 17]byte + var x14529 [1 << 17]byte + var x14530 [1 << 17]byte + var x14531 [1 << 17]byte + var x14532 [1 << 17]byte + var x14533 [1 << 17]byte + var x14534 [1 << 17]byte + var x14535 [1 << 17]byte + var x14536 [1 << 17]byte + var x14537 [1 << 17]byte + var x14538 [1 << 17]byte + var x14539 [1 << 17]byte + var x14540 [1 << 17]byte + var x14541 [1 << 17]byte + var x14542 [1 << 17]byte + var x14543 [1 << 17]byte + var x14544 [1 << 17]byte + var x14545 [1 << 17]byte + var x14546 [1 << 17]byte + var x14547 [1 << 17]byte + var x14548 [1 << 17]byte + var x14549 [1 << 17]byte + var x14550 [1 << 17]byte + var x14551 [1 << 17]byte + var x14552 [1 << 17]byte + var x14553 [1 << 17]byte + var x14554 [1 << 17]byte + var x14555 [1 << 17]byte + var x14556 [1 << 17]byte + var x14557 [1 << 17]byte + var x14558 [1 << 17]byte + var x14559 [1 << 17]byte + var x14560 [1 << 17]byte + var x14561 [1 << 17]byte + var x14562 [1 << 17]byte + var x14563 [1 << 17]byte + var x14564 [1 << 17]byte + var x14565 [1 << 17]byte + var x14566 [1 << 17]byte + var x14567 [1 << 17]byte + var x14568 [1 << 17]byte + var x14569 [1 << 17]byte + var x14570 [1 << 17]byte + var x14571 [1 << 17]byte + var x14572 [1 << 17]byte + var x14573 [1 << 17]byte + var x14574 [1 << 17]byte + var x14575 [1 << 17]byte + var x14576 [1 << 17]byte + var x14577 [1 << 17]byte + var x14578 [1 << 17]byte + var x14579 [1 << 17]byte + var x14580 [1 << 17]byte + var x14581 [1 << 17]byte + var x14582 [1 << 17]byte + var x14583 [1 << 17]byte + var x14584 [1 << 17]byte + var x14585 [1 << 17]byte + var x14586 [1 << 17]byte + var x14587 [1 << 17]byte + var x14588 [1 << 17]byte + var x14589 [1 << 17]byte + var x14590 [1 << 17]byte + var x14591 [1 << 17]byte + var x14592 [1 << 17]byte + var x14593 [1 << 17]byte + var x14594 [1 << 17]byte + var x14595 [1 << 17]byte + var x14596 [1 << 17]byte + var x14597 [1 << 17]byte + var x14598 [1 << 17]byte + var x14599 [1 << 17]byte + var x14600 [1 << 17]byte + var x14601 [1 << 17]byte + var x14602 [1 << 17]byte + var x14603 [1 << 17]byte + var x14604 [1 << 17]byte + var x14605 [1 << 17]byte + var x14606 [1 << 17]byte + var x14607 [1 << 17]byte + var x14608 [1 << 17]byte + var x14609 [1 << 17]byte + var x14610 [1 << 17]byte + var x14611 [1 << 17]byte + var x14612 [1 << 17]byte + var x14613 [1 << 17]byte + var x14614 [1 << 17]byte + var x14615 [1 << 17]byte + var x14616 [1 << 17]byte + var x14617 [1 << 17]byte + var x14618 [1 << 17]byte + var x14619 [1 << 17]byte + var x14620 [1 << 17]byte + var x14621 [1 << 17]byte + var x14622 [1 << 17]byte + var x14623 [1 << 17]byte + var x14624 [1 << 17]byte + var x14625 [1 << 17]byte + var x14626 [1 << 17]byte + var x14627 [1 << 17]byte + var x14628 [1 << 17]byte + var x14629 [1 << 17]byte + var x14630 [1 << 17]byte + var x14631 [1 << 17]byte + var x14632 [1 << 17]byte + var x14633 [1 << 17]byte + var x14634 [1 << 17]byte + var x14635 [1 << 17]byte + var x14636 [1 << 17]byte + var x14637 [1 << 17]byte + var x14638 [1 << 17]byte + var x14639 [1 << 17]byte + var x14640 [1 << 17]byte + var x14641 [1 << 17]byte + var x14642 [1 << 17]byte + var x14643 [1 << 17]byte + var x14644 [1 << 17]byte + var x14645 [1 << 17]byte + var x14646 [1 << 17]byte + var x14647 [1 << 17]byte + var x14648 [1 << 17]byte + var x14649 [1 << 17]byte + var x14650 [1 << 17]byte + var x14651 [1 << 17]byte + var x14652 [1 << 17]byte + var x14653 [1 << 17]byte + var x14654 [1 << 17]byte + var x14655 [1 << 17]byte + var x14656 [1 << 17]byte + var x14657 [1 << 17]byte + var x14658 [1 << 17]byte + var x14659 [1 << 17]byte + var x14660 [1 << 17]byte + var x14661 [1 << 17]byte + var x14662 [1 << 17]byte + var x14663 [1 << 17]byte + var x14664 [1 << 17]byte + var x14665 [1 << 17]byte + var x14666 [1 << 17]byte + var x14667 [1 << 17]byte + var x14668 [1 << 17]byte + var x14669 [1 << 17]byte + var x14670 [1 << 17]byte + var x14671 [1 << 17]byte + var x14672 [1 << 17]byte + var x14673 [1 << 17]byte + var x14674 [1 << 17]byte + var x14675 [1 << 17]byte + var x14676 [1 << 17]byte + var x14677 [1 << 17]byte + var x14678 [1 << 17]byte + var x14679 [1 << 17]byte + var x14680 [1 << 17]byte + var x14681 [1 << 17]byte + var x14682 [1 << 17]byte + var x14683 [1 << 17]byte + var x14684 [1 << 17]byte + var x14685 [1 << 17]byte + var x14686 [1 << 17]byte + var x14687 [1 << 17]byte + var x14688 [1 << 17]byte + var x14689 [1 << 17]byte + var x14690 [1 << 17]byte + var x14691 [1 << 17]byte + var x14692 [1 << 17]byte + var x14693 [1 << 17]byte + var x14694 [1 << 17]byte + var x14695 [1 << 17]byte + var x14696 [1 << 17]byte + var x14697 [1 << 17]byte + var x14698 [1 << 17]byte + var x14699 [1 << 17]byte + var x14700 [1 << 17]byte + var x14701 [1 << 17]byte + var x14702 [1 << 17]byte + var x14703 [1 << 17]byte + var x14704 [1 << 17]byte + var x14705 [1 << 17]byte + var x14706 [1 << 17]byte + var x14707 [1 << 17]byte + var x14708 [1 << 17]byte + var x14709 [1 << 17]byte + var x14710 [1 << 17]byte + var x14711 [1 << 17]byte + var x14712 [1 << 17]byte + var x14713 [1 << 17]byte + var x14714 [1 << 17]byte + var x14715 [1 << 17]byte + var x14716 [1 << 17]byte + var x14717 [1 << 17]byte + var x14718 [1 << 17]byte + var x14719 [1 << 17]byte + var x14720 [1 << 17]byte + var x14721 [1 << 17]byte + var x14722 [1 << 17]byte + var x14723 [1 << 17]byte + var x14724 [1 << 17]byte + var x14725 [1 << 17]byte + var x14726 [1 << 17]byte + var x14727 [1 << 17]byte + var x14728 [1 << 17]byte + var x14729 [1 << 17]byte + var x14730 [1 << 17]byte + var x14731 [1 << 17]byte + var x14732 [1 << 17]byte + var x14733 [1 << 17]byte + var x14734 [1 << 17]byte + var x14735 [1 << 17]byte + var x14736 [1 << 17]byte + var x14737 [1 << 17]byte + var x14738 [1 << 17]byte + var x14739 [1 << 17]byte + var x14740 [1 << 17]byte + var x14741 [1 << 17]byte + var x14742 [1 << 17]byte + var x14743 [1 << 17]byte + var x14744 [1 << 17]byte + var x14745 [1 << 17]byte + var x14746 [1 << 17]byte + var x14747 [1 << 17]byte + var x14748 [1 << 17]byte + var x14749 [1 << 17]byte + var x14750 [1 << 17]byte + var x14751 [1 << 17]byte + var x14752 [1 << 17]byte + var x14753 [1 << 17]byte + var x14754 [1 << 17]byte + var x14755 [1 << 17]byte + var x14756 [1 << 17]byte + var x14757 [1 << 17]byte + var x14758 [1 << 17]byte + var x14759 [1 << 17]byte + var x14760 [1 << 17]byte + var x14761 [1 << 17]byte + var x14762 [1 << 17]byte + var x14763 [1 << 17]byte + var x14764 [1 << 17]byte + var x14765 [1 << 17]byte + var x14766 [1 << 17]byte + var x14767 [1 << 17]byte + var x14768 [1 << 17]byte + var x14769 [1 << 17]byte + var x14770 [1 << 17]byte + var x14771 [1 << 17]byte + var x14772 [1 << 17]byte + var x14773 [1 << 17]byte + var x14774 [1 << 17]byte + var x14775 [1 << 17]byte + var x14776 [1 << 17]byte + var x14777 [1 << 17]byte + var x14778 [1 << 17]byte + var x14779 [1 << 17]byte + var x14780 [1 << 17]byte + var x14781 [1 << 17]byte + var x14782 [1 << 17]byte + var x14783 [1 << 17]byte + var x14784 [1 << 17]byte + var x14785 [1 << 17]byte + var x14786 [1 << 17]byte + var x14787 [1 << 17]byte + var x14788 [1 << 17]byte + var x14789 [1 << 17]byte + var x14790 [1 << 17]byte + var x14791 [1 << 17]byte + var x14792 [1 << 17]byte + var x14793 [1 << 17]byte + var x14794 [1 << 17]byte + var x14795 [1 << 17]byte + var x14796 [1 << 17]byte + var x14797 [1 << 17]byte + var x14798 [1 << 17]byte + var x14799 [1 << 17]byte + var x14800 [1 << 17]byte + var x14801 [1 << 17]byte + var x14802 [1 << 17]byte + var x14803 [1 << 17]byte + var x14804 [1 << 17]byte + var x14805 [1 << 17]byte + var x14806 [1 << 17]byte + var x14807 [1 << 17]byte + var x14808 [1 << 17]byte + var x14809 [1 << 17]byte + var x14810 [1 << 17]byte + var x14811 [1 << 17]byte + var x14812 [1 << 17]byte + var x14813 [1 << 17]byte + var x14814 [1 << 17]byte + var x14815 [1 << 17]byte + var x14816 [1 << 17]byte + var x14817 [1 << 17]byte + var x14818 [1 << 17]byte + var x14819 [1 << 17]byte + var x14820 [1 << 17]byte + var x14821 [1 << 17]byte + var x14822 [1 << 17]byte + var x14823 [1 << 17]byte + var x14824 [1 << 17]byte + var x14825 [1 << 17]byte + var x14826 [1 << 17]byte + var x14827 [1 << 17]byte + var x14828 [1 << 17]byte + var x14829 [1 << 17]byte + var x14830 [1 << 17]byte + var x14831 [1 << 17]byte + var x14832 [1 << 17]byte + var x14833 [1 << 17]byte + var x14834 [1 << 17]byte + var x14835 [1 << 17]byte + var x14836 [1 << 17]byte + var x14837 [1 << 17]byte + var x14838 [1 << 17]byte + var x14839 [1 << 17]byte + var x14840 [1 << 17]byte + var x14841 [1 << 17]byte + var x14842 [1 << 17]byte + var x14843 [1 << 17]byte + var x14844 [1 << 17]byte + var x14845 [1 << 17]byte + var x14846 [1 << 17]byte + var x14847 [1 << 17]byte + var x14848 [1 << 17]byte + var x14849 [1 << 17]byte + var x14850 [1 << 17]byte + var x14851 [1 << 17]byte + var x14852 [1 << 17]byte + var x14853 [1 << 17]byte + var x14854 [1 << 17]byte + var x14855 [1 << 17]byte + var x14856 [1 << 17]byte + var x14857 [1 << 17]byte + var x14858 [1 << 17]byte + var x14859 [1 << 17]byte + var x14860 [1 << 17]byte + var x14861 [1 << 17]byte + var x14862 [1 << 17]byte + var x14863 [1 << 17]byte + var x14864 [1 << 17]byte + var x14865 [1 << 17]byte + var x14866 [1 << 17]byte + var x14867 [1 << 17]byte + var x14868 [1 << 17]byte + var x14869 [1 << 17]byte + var x14870 [1 << 17]byte + var x14871 [1 << 17]byte + var x14872 [1 << 17]byte + var x14873 [1 << 17]byte + var x14874 [1 << 17]byte + var x14875 [1 << 17]byte + var x14876 [1 << 17]byte + var x14877 [1 << 17]byte + var x14878 [1 << 17]byte + var x14879 [1 << 17]byte + var x14880 [1 << 17]byte + var x14881 [1 << 17]byte + var x14882 [1 << 17]byte + var x14883 [1 << 17]byte + var x14884 [1 << 17]byte + var x14885 [1 << 17]byte + var x14886 [1 << 17]byte + var x14887 [1 << 17]byte + var x14888 [1 << 17]byte + var x14889 [1 << 17]byte + var x14890 [1 << 17]byte + var x14891 [1 << 17]byte + var x14892 [1 << 17]byte + var x14893 [1 << 17]byte + var x14894 [1 << 17]byte + var x14895 [1 << 17]byte + var x14896 [1 << 17]byte + var x14897 [1 << 17]byte + var x14898 [1 << 17]byte + var x14899 [1 << 17]byte + var x14900 [1 << 17]byte + var x14901 [1 << 17]byte + var x14902 [1 << 17]byte + var x14903 [1 << 17]byte + var x14904 [1 << 17]byte + var x14905 [1 << 17]byte + var x14906 [1 << 17]byte + var x14907 [1 << 17]byte + var x14908 [1 << 17]byte + var x14909 [1 << 17]byte + var x14910 [1 << 17]byte + var x14911 [1 << 17]byte + var x14912 [1 << 17]byte + var x14913 [1 << 17]byte + var x14914 [1 << 17]byte + var x14915 [1 << 17]byte + var x14916 [1 << 17]byte + var x14917 [1 << 17]byte + var x14918 [1 << 17]byte + var x14919 [1 << 17]byte + var x14920 [1 << 17]byte + var x14921 [1 << 17]byte + var x14922 [1 << 17]byte + var x14923 [1 << 17]byte + var x14924 [1 << 17]byte + var x14925 [1 << 17]byte + var x14926 [1 << 17]byte + var x14927 [1 << 17]byte + var x14928 [1 << 17]byte + var x14929 [1 << 17]byte + var x14930 [1 << 17]byte + var x14931 [1 << 17]byte + var x14932 [1 << 17]byte + var x14933 [1 << 17]byte + var x14934 [1 << 17]byte + var x14935 [1 << 17]byte + var x14936 [1 << 17]byte + var x14937 [1 << 17]byte + var x14938 [1 << 17]byte + var x14939 [1 << 17]byte + var x14940 [1 << 17]byte + var x14941 [1 << 17]byte + var x14942 [1 << 17]byte + var x14943 [1 << 17]byte + var x14944 [1 << 17]byte + var x14945 [1 << 17]byte + var x14946 [1 << 17]byte + var x14947 [1 << 17]byte + var x14948 [1 << 17]byte + var x14949 [1 << 17]byte + var x14950 [1 << 17]byte + var x14951 [1 << 17]byte + var x14952 [1 << 17]byte + var x14953 [1 << 17]byte + var x14954 [1 << 17]byte + var x14955 [1 << 17]byte + var x14956 [1 << 17]byte + var x14957 [1 << 17]byte + var x14958 [1 << 17]byte + var x14959 [1 << 17]byte + var x14960 [1 << 17]byte + var x14961 [1 << 17]byte + var x14962 [1 << 17]byte + var x14963 [1 << 17]byte + var x14964 [1 << 17]byte + var x14965 [1 << 17]byte + var x14966 [1 << 17]byte + var x14967 [1 << 17]byte + var x14968 [1 << 17]byte + var x14969 [1 << 17]byte + var x14970 [1 << 17]byte + var x14971 [1 << 17]byte + var x14972 [1 << 17]byte + var x14973 [1 << 17]byte + var x14974 [1 << 17]byte + var x14975 [1 << 17]byte + var x14976 [1 << 17]byte + var x14977 [1 << 17]byte + var x14978 [1 << 17]byte + var x14979 [1 << 17]byte + var x14980 [1 << 17]byte + var x14981 [1 << 17]byte + var x14982 [1 << 17]byte + var x14983 [1 << 17]byte + var x14984 [1 << 17]byte + var x14985 [1 << 17]byte + var x14986 [1 << 17]byte + var x14987 [1 << 17]byte + var x14988 [1 << 17]byte + var x14989 [1 << 17]byte + var x14990 [1 << 17]byte + var x14991 [1 << 17]byte + var x14992 [1 << 17]byte + var x14993 [1 << 17]byte + var x14994 [1 << 17]byte + var x14995 [1 << 17]byte + var x14996 [1 << 17]byte + var x14997 [1 << 17]byte + var x14998 [1 << 17]byte + var x14999 [1 << 17]byte + var x15000 [1 << 17]byte + var x15001 [1 << 17]byte + var x15002 [1 << 17]byte + var x15003 [1 << 17]byte + var x15004 [1 << 17]byte + var x15005 [1 << 17]byte + var x15006 [1 << 17]byte + var x15007 [1 << 17]byte + var x15008 [1 << 17]byte + var x15009 [1 << 17]byte + var x15010 [1 << 17]byte + var x15011 [1 << 17]byte + var x15012 [1 << 17]byte + var x15013 [1 << 17]byte + var x15014 [1 << 17]byte + var x15015 [1 << 17]byte + var x15016 [1 << 17]byte + var x15017 [1 << 17]byte + var x15018 [1 << 17]byte + var x15019 [1 << 17]byte + var x15020 [1 << 17]byte + var x15021 [1 << 17]byte + var x15022 [1 << 17]byte + var x15023 [1 << 17]byte + var x15024 [1 << 17]byte + var x15025 [1 << 17]byte + var x15026 [1 << 17]byte + var x15027 [1 << 17]byte + var x15028 [1 << 17]byte + var x15029 [1 << 17]byte + var x15030 [1 << 17]byte + var x15031 [1 << 17]byte + var x15032 [1 << 17]byte + var x15033 [1 << 17]byte + var x15034 [1 << 17]byte + var x15035 [1 << 17]byte + var x15036 [1 << 17]byte + var x15037 [1 << 17]byte + var x15038 [1 << 17]byte + var x15039 [1 << 17]byte + var x15040 [1 << 17]byte + var x15041 [1 << 17]byte + var x15042 [1 << 17]byte + var x15043 [1 << 17]byte + var x15044 [1 << 17]byte + var x15045 [1 << 17]byte + var x15046 [1 << 17]byte + var x15047 [1 << 17]byte + var x15048 [1 << 17]byte + var x15049 [1 << 17]byte + var x15050 [1 << 17]byte + var x15051 [1 << 17]byte + var x15052 [1 << 17]byte + var x15053 [1 << 17]byte + var x15054 [1 << 17]byte + var x15055 [1 << 17]byte + var x15056 [1 << 17]byte + var x15057 [1 << 17]byte + var x15058 [1 << 17]byte + var x15059 [1 << 17]byte + var x15060 [1 << 17]byte + var x15061 [1 << 17]byte + var x15062 [1 << 17]byte + var x15063 [1 << 17]byte + var x15064 [1 << 17]byte + var x15065 [1 << 17]byte + var x15066 [1 << 17]byte + var x15067 [1 << 17]byte + var x15068 [1 << 17]byte + var x15069 [1 << 17]byte + var x15070 [1 << 17]byte + var x15071 [1 << 17]byte + var x15072 [1 << 17]byte + var x15073 [1 << 17]byte + var x15074 [1 << 17]byte + var x15075 [1 << 17]byte + var x15076 [1 << 17]byte + var x15077 [1 << 17]byte + var x15078 [1 << 17]byte + var x15079 [1 << 17]byte + var x15080 [1 << 17]byte + var x15081 [1 << 17]byte + var x15082 [1 << 17]byte + var x15083 [1 << 17]byte + var x15084 [1 << 17]byte + var x15085 [1 << 17]byte + var x15086 [1 << 17]byte + var x15087 [1 << 17]byte + var x15088 [1 << 17]byte + var x15089 [1 << 17]byte + var x15090 [1 << 17]byte + var x15091 [1 << 17]byte + var x15092 [1 << 17]byte + var x15093 [1 << 17]byte + var x15094 [1 << 17]byte + var x15095 [1 << 17]byte + var x15096 [1 << 17]byte + var x15097 [1 << 17]byte + var x15098 [1 << 17]byte + var x15099 [1 << 17]byte + var x15100 [1 << 17]byte + var x15101 [1 << 17]byte + var x15102 [1 << 17]byte + var x15103 [1 << 17]byte + var x15104 [1 << 17]byte + var x15105 [1 << 17]byte + var x15106 [1 << 17]byte + var x15107 [1 << 17]byte + var x15108 [1 << 17]byte + var x15109 [1 << 17]byte + var x15110 [1 << 17]byte + var x15111 [1 << 17]byte + var x15112 [1 << 17]byte + var x15113 [1 << 17]byte + var x15114 [1 << 17]byte + var x15115 [1 << 17]byte + var x15116 [1 << 17]byte + var x15117 [1 << 17]byte + var x15118 [1 << 17]byte + var x15119 [1 << 17]byte + var x15120 [1 << 17]byte + var x15121 [1 << 17]byte + var x15122 [1 << 17]byte + var x15123 [1 << 17]byte + var x15124 [1 << 17]byte + var x15125 [1 << 17]byte + var x15126 [1 << 17]byte + var x15127 [1 << 17]byte + var x15128 [1 << 17]byte + var x15129 [1 << 17]byte + var x15130 [1 << 17]byte + var x15131 [1 << 17]byte + var x15132 [1 << 17]byte + var x15133 [1 << 17]byte + var x15134 [1 << 17]byte + var x15135 [1 << 17]byte + var x15136 [1 << 17]byte + var x15137 [1 << 17]byte + var x15138 [1 << 17]byte + var x15139 [1 << 17]byte + var x15140 [1 << 17]byte + var x15141 [1 << 17]byte + var x15142 [1 << 17]byte + var x15143 [1 << 17]byte + var x15144 [1 << 17]byte + var x15145 [1 << 17]byte + var x15146 [1 << 17]byte + var x15147 [1 << 17]byte + var x15148 [1 << 17]byte + var x15149 [1 << 17]byte + var x15150 [1 << 17]byte + var x15151 [1 << 17]byte + var x15152 [1 << 17]byte + var x15153 [1 << 17]byte + var x15154 [1 << 17]byte + var x15155 [1 << 17]byte + var x15156 [1 << 17]byte + var x15157 [1 << 17]byte + var x15158 [1 << 17]byte + var x15159 [1 << 17]byte + var x15160 [1 << 17]byte + var x15161 [1 << 17]byte + var x15162 [1 << 17]byte + var x15163 [1 << 17]byte + var x15164 [1 << 17]byte + var x15165 [1 << 17]byte + var x15166 [1 << 17]byte + var x15167 [1 << 17]byte + var x15168 [1 << 17]byte + var x15169 [1 << 17]byte + var x15170 [1 << 17]byte + var x15171 [1 << 17]byte + var x15172 [1 << 17]byte + var x15173 [1 << 17]byte + var x15174 [1 << 17]byte + var x15175 [1 << 17]byte + var x15176 [1 << 17]byte + var x15177 [1 << 17]byte + var x15178 [1 << 17]byte + var x15179 [1 << 17]byte + var x15180 [1 << 17]byte + var x15181 [1 << 17]byte + var x15182 [1 << 17]byte + var x15183 [1 << 17]byte + var x15184 [1 << 17]byte + var x15185 [1 << 17]byte + var x15186 [1 << 17]byte + var x15187 [1 << 17]byte + var x15188 [1 << 17]byte + var x15189 [1 << 17]byte + var x15190 [1 << 17]byte + var x15191 [1 << 17]byte + var x15192 [1 << 17]byte + var x15193 [1 << 17]byte + var x15194 [1 << 17]byte + var x15195 [1 << 17]byte + var x15196 [1 << 17]byte + var x15197 [1 << 17]byte + var x15198 [1 << 17]byte + var x15199 [1 << 17]byte + var x15200 [1 << 17]byte + var x15201 [1 << 17]byte + var x15202 [1 << 17]byte + var x15203 [1 << 17]byte + var x15204 [1 << 17]byte + var x15205 [1 << 17]byte + var x15206 [1 << 17]byte + var x15207 [1 << 17]byte + var x15208 [1 << 17]byte + var x15209 [1 << 17]byte + var x15210 [1 << 17]byte + var x15211 [1 << 17]byte + var x15212 [1 << 17]byte + var x15213 [1 << 17]byte + var x15214 [1 << 17]byte + var x15215 [1 << 17]byte + var x15216 [1 << 17]byte + var x15217 [1 << 17]byte + var x15218 [1 << 17]byte + var x15219 [1 << 17]byte + var x15220 [1 << 17]byte + var x15221 [1 << 17]byte + var x15222 [1 << 17]byte + var x15223 [1 << 17]byte + var x15224 [1 << 17]byte + var x15225 [1 << 17]byte + var x15226 [1 << 17]byte + var x15227 [1 << 17]byte + var x15228 [1 << 17]byte + var x15229 [1 << 17]byte + var x15230 [1 << 17]byte + var x15231 [1 << 17]byte + var x15232 [1 << 17]byte + var x15233 [1 << 17]byte + var x15234 [1 << 17]byte + var x15235 [1 << 17]byte + var x15236 [1 << 17]byte + var x15237 [1 << 17]byte + var x15238 [1 << 17]byte + var x15239 [1 << 17]byte + var x15240 [1 << 17]byte + var x15241 [1 << 17]byte + var x15242 [1 << 17]byte + var x15243 [1 << 17]byte + var x15244 [1 << 17]byte + var x15245 [1 << 17]byte + var x15246 [1 << 17]byte + var x15247 [1 << 17]byte + var x15248 [1 << 17]byte + var x15249 [1 << 17]byte + var x15250 [1 << 17]byte + var x15251 [1 << 17]byte + var x15252 [1 << 17]byte + var x15253 [1 << 17]byte + var x15254 [1 << 17]byte + var x15255 [1 << 17]byte + var x15256 [1 << 17]byte + var x15257 [1 << 17]byte + var x15258 [1 << 17]byte + var x15259 [1 << 17]byte + var x15260 [1 << 17]byte + var x15261 [1 << 17]byte + var x15262 [1 << 17]byte + var x15263 [1 << 17]byte + var x15264 [1 << 17]byte + var x15265 [1 << 17]byte + var x15266 [1 << 17]byte + var x15267 [1 << 17]byte + var x15268 [1 << 17]byte + var x15269 [1 << 17]byte + var x15270 [1 << 17]byte + var x15271 [1 << 17]byte + var x15272 [1 << 17]byte + var x15273 [1 << 17]byte + var x15274 [1 << 17]byte + var x15275 [1 << 17]byte + var x15276 [1 << 17]byte + var x15277 [1 << 17]byte + var x15278 [1 << 17]byte + var x15279 [1 << 17]byte + var x15280 [1 << 17]byte + var x15281 [1 << 17]byte + var x15282 [1 << 17]byte + var x15283 [1 << 17]byte + var x15284 [1 << 17]byte + var x15285 [1 << 17]byte + var x15286 [1 << 17]byte + var x15287 [1 << 17]byte + var x15288 [1 << 17]byte + var x15289 [1 << 17]byte + var x15290 [1 << 17]byte + var x15291 [1 << 17]byte + var x15292 [1 << 17]byte + var x15293 [1 << 17]byte + var x15294 [1 << 17]byte + var x15295 [1 << 17]byte + var x15296 [1 << 17]byte + var x15297 [1 << 17]byte + var x15298 [1 << 17]byte + var x15299 [1 << 17]byte + var x15300 [1 << 17]byte + var x15301 [1 << 17]byte + var x15302 [1 << 17]byte + var x15303 [1 << 17]byte + var x15304 [1 << 17]byte + var x15305 [1 << 17]byte + var x15306 [1 << 17]byte + var x15307 [1 << 17]byte + var x15308 [1 << 17]byte + var x15309 [1 << 17]byte + var x15310 [1 << 17]byte + var x15311 [1 << 17]byte + var x15312 [1 << 17]byte + var x15313 [1 << 17]byte + var x15314 [1 << 17]byte + var x15315 [1 << 17]byte + var x15316 [1 << 17]byte + var x15317 [1 << 17]byte + var x15318 [1 << 17]byte + var x15319 [1 << 17]byte + var x15320 [1 << 17]byte + var x15321 [1 << 17]byte + var x15322 [1 << 17]byte + var x15323 [1 << 17]byte + var x15324 [1 << 17]byte + var x15325 [1 << 17]byte + var x15326 [1 << 17]byte + var x15327 [1 << 17]byte + var x15328 [1 << 17]byte + var x15329 [1 << 17]byte + var x15330 [1 << 17]byte + var x15331 [1 << 17]byte + var x15332 [1 << 17]byte + var x15333 [1 << 17]byte + var x15334 [1 << 17]byte + var x15335 [1 << 17]byte + var x15336 [1 << 17]byte + var x15337 [1 << 17]byte + var x15338 [1 << 17]byte + var x15339 [1 << 17]byte + var x15340 [1 << 17]byte + var x15341 [1 << 17]byte + var x15342 [1 << 17]byte + var x15343 [1 << 17]byte + var x15344 [1 << 17]byte + var x15345 [1 << 17]byte + var x15346 [1 << 17]byte + var x15347 [1 << 17]byte + var x15348 [1 << 17]byte + var x15349 [1 << 17]byte + var x15350 [1 << 17]byte + var x15351 [1 << 17]byte + var x15352 [1 << 17]byte + var x15353 [1 << 17]byte + var x15354 [1 << 17]byte + var x15355 [1 << 17]byte + var x15356 [1 << 17]byte + var x15357 [1 << 17]byte + var x15358 [1 << 17]byte + var x15359 [1 << 17]byte + var x15360 [1 << 17]byte + var x15361 [1 << 17]byte + var x15362 [1 << 17]byte + var x15363 [1 << 17]byte + var x15364 [1 << 17]byte + var x15365 [1 << 17]byte + var x15366 [1 << 17]byte + var x15367 [1 << 17]byte + var x15368 [1 << 17]byte + var x15369 [1 << 17]byte + var x15370 [1 << 17]byte + var x15371 [1 << 17]byte + var x15372 [1 << 17]byte + var x15373 [1 << 17]byte + var x15374 [1 << 17]byte + var x15375 [1 << 17]byte + var x15376 [1 << 17]byte + var x15377 [1 << 17]byte + var x15378 [1 << 17]byte + var x15379 [1 << 17]byte + var x15380 [1 << 17]byte + var x15381 [1 << 17]byte + var x15382 [1 << 17]byte + var x15383 [1 << 17]byte + var x15384 [1 << 17]byte + var x15385 [1 << 17]byte + var x15386 [1 << 17]byte + var x15387 [1 << 17]byte + var x15388 [1 << 17]byte + var x15389 [1 << 17]byte + var x15390 [1 << 17]byte + var x15391 [1 << 17]byte + var x15392 [1 << 17]byte + var x15393 [1 << 17]byte + var x15394 [1 << 17]byte + var x15395 [1 << 17]byte + var x15396 [1 << 17]byte + var x15397 [1 << 17]byte + var x15398 [1 << 17]byte + var x15399 [1 << 17]byte + var x15400 [1 << 17]byte + var x15401 [1 << 17]byte + var x15402 [1 << 17]byte + var x15403 [1 << 17]byte + var x15404 [1 << 17]byte + var x15405 [1 << 17]byte + var x15406 [1 << 17]byte + var x15407 [1 << 17]byte + var x15408 [1 << 17]byte + var x15409 [1 << 17]byte + var x15410 [1 << 17]byte + var x15411 [1 << 17]byte + var x15412 [1 << 17]byte + var x15413 [1 << 17]byte + var x15414 [1 << 17]byte + var x15415 [1 << 17]byte + var x15416 [1 << 17]byte + var x15417 [1 << 17]byte + var x15418 [1 << 17]byte + var x15419 [1 << 17]byte + var x15420 [1 << 17]byte + var x15421 [1 << 17]byte + var x15422 [1 << 17]byte + var x15423 [1 << 17]byte + var x15424 [1 << 17]byte + var x15425 [1 << 17]byte + var x15426 [1 << 17]byte + var x15427 [1 << 17]byte + var x15428 [1 << 17]byte + var x15429 [1 << 17]byte + var x15430 [1 << 17]byte + var x15431 [1 << 17]byte + var x15432 [1 << 17]byte + var x15433 [1 << 17]byte + var x15434 [1 << 17]byte + var x15435 [1 << 17]byte + var x15436 [1 << 17]byte + var x15437 [1 << 17]byte + var x15438 [1 << 17]byte + var x15439 [1 << 17]byte + var x15440 [1 << 17]byte + var x15441 [1 << 17]byte + var x15442 [1 << 17]byte + var x15443 [1 << 17]byte + var x15444 [1 << 17]byte + var x15445 [1 << 17]byte + var x15446 [1 << 17]byte + var x15447 [1 << 17]byte + var x15448 [1 << 17]byte + var x15449 [1 << 17]byte + var x15450 [1 << 17]byte + var x15451 [1 << 17]byte + var x15452 [1 << 17]byte + var x15453 [1 << 17]byte + var x15454 [1 << 17]byte + var x15455 [1 << 17]byte + var x15456 [1 << 17]byte + var x15457 [1 << 17]byte + var x15458 [1 << 17]byte + var x15459 [1 << 17]byte + var x15460 [1 << 17]byte + var x15461 [1 << 17]byte + var x15462 [1 << 17]byte + var x15463 [1 << 17]byte + var x15464 [1 << 17]byte + var x15465 [1 << 17]byte + var x15466 [1 << 17]byte + var x15467 [1 << 17]byte + var x15468 [1 << 17]byte + var x15469 [1 << 17]byte + var x15470 [1 << 17]byte + var x15471 [1 << 17]byte + var x15472 [1 << 17]byte + var x15473 [1 << 17]byte + var x15474 [1 << 17]byte + var x15475 [1 << 17]byte + var x15476 [1 << 17]byte + var x15477 [1 << 17]byte + var x15478 [1 << 17]byte + var x15479 [1 << 17]byte + var x15480 [1 << 17]byte + var x15481 [1 << 17]byte + var x15482 [1 << 17]byte + var x15483 [1 << 17]byte + var x15484 [1 << 17]byte + var x15485 [1 << 17]byte + var x15486 [1 << 17]byte + var x15487 [1 << 17]byte + var x15488 [1 << 17]byte + var x15489 [1 << 17]byte + var x15490 [1 << 17]byte + var x15491 [1 << 17]byte + var x15492 [1 << 17]byte + var x15493 [1 << 17]byte + var x15494 [1 << 17]byte + var x15495 [1 << 17]byte + var x15496 [1 << 17]byte + var x15497 [1 << 17]byte + var x15498 [1 << 17]byte + var x15499 [1 << 17]byte + var x15500 [1 << 17]byte + var x15501 [1 << 17]byte + var x15502 [1 << 17]byte + var x15503 [1 << 17]byte + var x15504 [1 << 17]byte + var x15505 [1 << 17]byte + var x15506 [1 << 17]byte + var x15507 [1 << 17]byte + var x15508 [1 << 17]byte + var x15509 [1 << 17]byte + var x15510 [1 << 17]byte + var x15511 [1 << 17]byte + var x15512 [1 << 17]byte + var x15513 [1 << 17]byte + var x15514 [1 << 17]byte + var x15515 [1 << 17]byte + var x15516 [1 << 17]byte + var x15517 [1 << 17]byte + var x15518 [1 << 17]byte + var x15519 [1 << 17]byte + var x15520 [1 << 17]byte + var x15521 [1 << 17]byte + var x15522 [1 << 17]byte + var x15523 [1 << 17]byte + var x15524 [1 << 17]byte + var x15525 [1 << 17]byte + var x15526 [1 << 17]byte + var x15527 [1 << 17]byte + var x15528 [1 << 17]byte + var x15529 [1 << 17]byte + var x15530 [1 << 17]byte + var x15531 [1 << 17]byte + var x15532 [1 << 17]byte + var x15533 [1 << 17]byte + var x15534 [1 << 17]byte + var x15535 [1 << 17]byte + var x15536 [1 << 17]byte + var x15537 [1 << 17]byte + var x15538 [1 << 17]byte + var x15539 [1 << 17]byte + var x15540 [1 << 17]byte + var x15541 [1 << 17]byte + var x15542 [1 << 17]byte + var x15543 [1 << 17]byte + var x15544 [1 << 17]byte + var x15545 [1 << 17]byte + var x15546 [1 << 17]byte + var x15547 [1 << 17]byte + var x15548 [1 << 17]byte + var x15549 [1 << 17]byte + var x15550 [1 << 17]byte + var x15551 [1 << 17]byte + var x15552 [1 << 17]byte + var x15553 [1 << 17]byte + var x15554 [1 << 17]byte + var x15555 [1 << 17]byte + var x15556 [1 << 17]byte + var x15557 [1 << 17]byte + var x15558 [1 << 17]byte + var x15559 [1 << 17]byte + var x15560 [1 << 17]byte + var x15561 [1 << 17]byte + var x15562 [1 << 17]byte + var x15563 [1 << 17]byte + var x15564 [1 << 17]byte + var x15565 [1 << 17]byte + var x15566 [1 << 17]byte + var x15567 [1 << 17]byte + var x15568 [1 << 17]byte + var x15569 [1 << 17]byte + var x15570 [1 << 17]byte + var x15571 [1 << 17]byte + var x15572 [1 << 17]byte + var x15573 [1 << 17]byte + var x15574 [1 << 17]byte + var x15575 [1 << 17]byte + var x15576 [1 << 17]byte + var x15577 [1 << 17]byte + var x15578 [1 << 17]byte + var x15579 [1 << 17]byte + var x15580 [1 << 17]byte + var x15581 [1 << 17]byte + var x15582 [1 << 17]byte + var x15583 [1 << 17]byte + var x15584 [1 << 17]byte + var x15585 [1 << 17]byte + var x15586 [1 << 17]byte + var x15587 [1 << 17]byte + var x15588 [1 << 17]byte + var x15589 [1 << 17]byte + var x15590 [1 << 17]byte + var x15591 [1 << 17]byte + var x15592 [1 << 17]byte + var x15593 [1 << 17]byte + var x15594 [1 << 17]byte + var x15595 [1 << 17]byte + var x15596 [1 << 17]byte + var x15597 [1 << 17]byte + var x15598 [1 << 17]byte + var x15599 [1 << 17]byte + var x15600 [1 << 17]byte + var x15601 [1 << 17]byte + var x15602 [1 << 17]byte + var x15603 [1 << 17]byte + var x15604 [1 << 17]byte + var x15605 [1 << 17]byte + var x15606 [1 << 17]byte + var x15607 [1 << 17]byte + var x15608 [1 << 17]byte + var x15609 [1 << 17]byte + var x15610 [1 << 17]byte + var x15611 [1 << 17]byte + var x15612 [1 << 17]byte + var x15613 [1 << 17]byte + var x15614 [1 << 17]byte + var x15615 [1 << 17]byte + var x15616 [1 << 17]byte + var x15617 [1 << 17]byte + var x15618 [1 << 17]byte + var x15619 [1 << 17]byte + var x15620 [1 << 17]byte + var x15621 [1 << 17]byte + var x15622 [1 << 17]byte + var x15623 [1 << 17]byte + var x15624 [1 << 17]byte + var x15625 [1 << 17]byte + var x15626 [1 << 17]byte + var x15627 [1 << 17]byte + var x15628 [1 << 17]byte + var x15629 [1 << 17]byte + var x15630 [1 << 17]byte + var x15631 [1 << 17]byte + var x15632 [1 << 17]byte + var x15633 [1 << 17]byte + var x15634 [1 << 17]byte + var x15635 [1 << 17]byte + var x15636 [1 << 17]byte + var x15637 [1 << 17]byte + var x15638 [1 << 17]byte + var x15639 [1 << 17]byte + var x15640 [1 << 17]byte + var x15641 [1 << 17]byte + var x15642 [1 << 17]byte + var x15643 [1 << 17]byte + var x15644 [1 << 17]byte + var x15645 [1 << 17]byte + var x15646 [1 << 17]byte + var x15647 [1 << 17]byte + var x15648 [1 << 17]byte + var x15649 [1 << 17]byte + var x15650 [1 << 17]byte + var x15651 [1 << 17]byte + var x15652 [1 << 17]byte + var x15653 [1 << 17]byte + var x15654 [1 << 17]byte + var x15655 [1 << 17]byte + var x15656 [1 << 17]byte + var x15657 [1 << 17]byte + var x15658 [1 << 17]byte + var x15659 [1 << 17]byte + var x15660 [1 << 17]byte + var x15661 [1 << 17]byte + var x15662 [1 << 17]byte + var x15663 [1 << 17]byte + var x15664 [1 << 17]byte + var x15665 [1 << 17]byte + var x15666 [1 << 17]byte + var x15667 [1 << 17]byte + var x15668 [1 << 17]byte + var x15669 [1 << 17]byte + var x15670 [1 << 17]byte + var x15671 [1 << 17]byte + var x15672 [1 << 17]byte + var x15673 [1 << 17]byte + var x15674 [1 << 17]byte + var x15675 [1 << 17]byte + var x15676 [1 << 17]byte + var x15677 [1 << 17]byte + var x15678 [1 << 17]byte + var x15679 [1 << 17]byte + var x15680 [1 << 17]byte + var x15681 [1 << 17]byte + var x15682 [1 << 17]byte + var x15683 [1 << 17]byte + var x15684 [1 << 17]byte + var x15685 [1 << 17]byte + var x15686 [1 << 17]byte + var x15687 [1 << 17]byte + var x15688 [1 << 17]byte + var x15689 [1 << 17]byte + var x15690 [1 << 17]byte + var x15691 [1 << 17]byte + var x15692 [1 << 17]byte + var x15693 [1 << 17]byte + var x15694 [1 << 17]byte + var x15695 [1 << 17]byte + var x15696 [1 << 17]byte + var x15697 [1 << 17]byte + var x15698 [1 << 17]byte + var x15699 [1 << 17]byte + var x15700 [1 << 17]byte + var x15701 [1 << 17]byte + var x15702 [1 << 17]byte + var x15703 [1 << 17]byte + var x15704 [1 << 17]byte + var x15705 [1 << 17]byte + var x15706 [1 << 17]byte + var x15707 [1 << 17]byte + var x15708 [1 << 17]byte + var x15709 [1 << 17]byte + var x15710 [1 << 17]byte + var x15711 [1 << 17]byte + var x15712 [1 << 17]byte + var x15713 [1 << 17]byte + var x15714 [1 << 17]byte + var x15715 [1 << 17]byte + var x15716 [1 << 17]byte + var x15717 [1 << 17]byte + var x15718 [1 << 17]byte + var x15719 [1 << 17]byte + var x15720 [1 << 17]byte + var x15721 [1 << 17]byte + var x15722 [1 << 17]byte + var x15723 [1 << 17]byte + var x15724 [1 << 17]byte + var x15725 [1 << 17]byte + var x15726 [1 << 17]byte + var x15727 [1 << 17]byte + var x15728 [1 << 17]byte + var x15729 [1 << 17]byte + var x15730 [1 << 17]byte + var x15731 [1 << 17]byte + var x15732 [1 << 17]byte + var x15733 [1 << 17]byte + var x15734 [1 << 17]byte + var x15735 [1 << 17]byte + var x15736 [1 << 17]byte + var x15737 [1 << 17]byte + var x15738 [1 << 17]byte + var x15739 [1 << 17]byte + var x15740 [1 << 17]byte + var x15741 [1 << 17]byte + var x15742 [1 << 17]byte + var x15743 [1 << 17]byte + var x15744 [1 << 17]byte + var x15745 [1 << 17]byte + var x15746 [1 << 17]byte + var x15747 [1 << 17]byte + var x15748 [1 << 17]byte + var x15749 [1 << 17]byte + var x15750 [1 << 17]byte + var x15751 [1 << 17]byte + var x15752 [1 << 17]byte + var x15753 [1 << 17]byte + var x15754 [1 << 17]byte + var x15755 [1 << 17]byte + var x15756 [1 << 17]byte + var x15757 [1 << 17]byte + var x15758 [1 << 17]byte + var x15759 [1 << 17]byte + var x15760 [1 << 17]byte + var x15761 [1 << 17]byte + var x15762 [1 << 17]byte + var x15763 [1 << 17]byte + var x15764 [1 << 17]byte + var x15765 [1 << 17]byte + var x15766 [1 << 17]byte + var x15767 [1 << 17]byte + var x15768 [1 << 17]byte + var x15769 [1 << 17]byte + var x15770 [1 << 17]byte + var x15771 [1 << 17]byte + var x15772 [1 << 17]byte + var x15773 [1 << 17]byte + var x15774 [1 << 17]byte + var x15775 [1 << 17]byte + var x15776 [1 << 17]byte + var x15777 [1 << 17]byte + var x15778 [1 << 17]byte + var x15779 [1 << 17]byte + var x15780 [1 << 17]byte + var x15781 [1 << 17]byte + var x15782 [1 << 17]byte + var x15783 [1 << 17]byte + var x15784 [1 << 17]byte + var x15785 [1 << 17]byte + var x15786 [1 << 17]byte + var x15787 [1 << 17]byte + var x15788 [1 << 17]byte + var x15789 [1 << 17]byte + var x15790 [1 << 17]byte + var x15791 [1 << 17]byte + var x15792 [1 << 17]byte + var x15793 [1 << 17]byte + var x15794 [1 << 17]byte + var x15795 [1 << 17]byte + var x15796 [1 << 17]byte + var x15797 [1 << 17]byte + var x15798 [1 << 17]byte + var x15799 [1 << 17]byte + var x15800 [1 << 17]byte + var x15801 [1 << 17]byte + var x15802 [1 << 17]byte + var x15803 [1 << 17]byte + var x15804 [1 << 17]byte + var x15805 [1 << 17]byte + var x15806 [1 << 17]byte + var x15807 [1 << 17]byte + var x15808 [1 << 17]byte + var x15809 [1 << 17]byte + var x15810 [1 << 17]byte + var x15811 [1 << 17]byte + var x15812 [1 << 17]byte + var x15813 [1 << 17]byte + var x15814 [1 << 17]byte + var x15815 [1 << 17]byte + var x15816 [1 << 17]byte + var x15817 [1 << 17]byte + var x15818 [1 << 17]byte + var x15819 [1 << 17]byte + var x15820 [1 << 17]byte + var x15821 [1 << 17]byte + var x15822 [1 << 17]byte + var x15823 [1 << 17]byte + var x15824 [1 << 17]byte + var x15825 [1 << 17]byte + var x15826 [1 << 17]byte + var x15827 [1 << 17]byte + var x15828 [1 << 17]byte + var x15829 [1 << 17]byte + var x15830 [1 << 17]byte + var x15831 [1 << 17]byte + var x15832 [1 << 17]byte + var x15833 [1 << 17]byte + var x15834 [1 << 17]byte + var x15835 [1 << 17]byte + var x15836 [1 << 17]byte + var x15837 [1 << 17]byte + var x15838 [1 << 17]byte + var x15839 [1 << 17]byte + var x15840 [1 << 17]byte + var x15841 [1 << 17]byte + var x15842 [1 << 17]byte + var x15843 [1 << 17]byte + var x15844 [1 << 17]byte + var x15845 [1 << 17]byte + var x15846 [1 << 17]byte + var x15847 [1 << 17]byte + var x15848 [1 << 17]byte + var x15849 [1 << 17]byte + var x15850 [1 << 17]byte + var x15851 [1 << 17]byte + var x15852 [1 << 17]byte + var x15853 [1 << 17]byte + var x15854 [1 << 17]byte + var x15855 [1 << 17]byte + var x15856 [1 << 17]byte + var x15857 [1 << 17]byte + var x15858 [1 << 17]byte + var x15859 [1 << 17]byte + var x15860 [1 << 17]byte + var x15861 [1 << 17]byte + var x15862 [1 << 17]byte + var x15863 [1 << 17]byte + var x15864 [1 << 17]byte + var x15865 [1 << 17]byte + var x15866 [1 << 17]byte + var x15867 [1 << 17]byte + var x15868 [1 << 17]byte + var x15869 [1 << 17]byte + var x15870 [1 << 17]byte + var x15871 [1 << 17]byte + var x15872 [1 << 17]byte + var x15873 [1 << 17]byte + var x15874 [1 << 17]byte + var x15875 [1 << 17]byte + var x15876 [1 << 17]byte + var x15877 [1 << 17]byte + var x15878 [1 << 17]byte + var x15879 [1 << 17]byte + var x15880 [1 << 17]byte + var x15881 [1 << 17]byte + var x15882 [1 << 17]byte + var x15883 [1 << 17]byte + var x15884 [1 << 17]byte + var x15885 [1 << 17]byte + var x15886 [1 << 17]byte + var x15887 [1 << 17]byte + var x15888 [1 << 17]byte + var x15889 [1 << 17]byte + var x15890 [1 << 17]byte + var x15891 [1 << 17]byte + var x15892 [1 << 17]byte + var x15893 [1 << 17]byte + var x15894 [1 << 17]byte + var x15895 [1 << 17]byte + var x15896 [1 << 17]byte + var x15897 [1 << 17]byte + var x15898 [1 << 17]byte + var x15899 [1 << 17]byte + var x15900 [1 << 17]byte + var x15901 [1 << 17]byte + var x15902 [1 << 17]byte + var x15903 [1 << 17]byte + var x15904 [1 << 17]byte + var x15905 [1 << 17]byte + var x15906 [1 << 17]byte + var x15907 [1 << 17]byte + var x15908 [1 << 17]byte + var x15909 [1 << 17]byte + var x15910 [1 << 17]byte + var x15911 [1 << 17]byte + var x15912 [1 << 17]byte + var x15913 [1 << 17]byte + var x15914 [1 << 17]byte + var x15915 [1 << 17]byte + var x15916 [1 << 17]byte + var x15917 [1 << 17]byte + var x15918 [1 << 17]byte + var x15919 [1 << 17]byte + var x15920 [1 << 17]byte + var x15921 [1 << 17]byte + var x15922 [1 << 17]byte + var x15923 [1 << 17]byte + var x15924 [1 << 17]byte + var x15925 [1 << 17]byte + var x15926 [1 << 17]byte + var x15927 [1 << 17]byte + var x15928 [1 << 17]byte + var x15929 [1 << 17]byte + var x15930 [1 << 17]byte + var x15931 [1 << 17]byte + var x15932 [1 << 17]byte + var x15933 [1 << 17]byte + var x15934 [1 << 17]byte + var x15935 [1 << 17]byte + var x15936 [1 << 17]byte + var x15937 [1 << 17]byte + var x15938 [1 << 17]byte + var x15939 [1 << 17]byte + var x15940 [1 << 17]byte + var x15941 [1 << 17]byte + var x15942 [1 << 17]byte + var x15943 [1 << 17]byte + var x15944 [1 << 17]byte + var x15945 [1 << 17]byte + var x15946 [1 << 17]byte + var x15947 [1 << 17]byte + var x15948 [1 << 17]byte + var x15949 [1 << 17]byte + var x15950 [1 << 17]byte + var x15951 [1 << 17]byte + var x15952 [1 << 17]byte + var x15953 [1 << 17]byte + var x15954 [1 << 17]byte + var x15955 [1 << 17]byte + var x15956 [1 << 17]byte + var x15957 [1 << 17]byte + var x15958 [1 << 17]byte + var x15959 [1 << 17]byte + var x15960 [1 << 17]byte + var x15961 [1 << 17]byte + var x15962 [1 << 17]byte + var x15963 [1 << 17]byte + var x15964 [1 << 17]byte + var x15965 [1 << 17]byte + var x15966 [1 << 17]byte + var x15967 [1 << 17]byte + var x15968 [1 << 17]byte + var x15969 [1 << 17]byte + var x15970 [1 << 17]byte + var x15971 [1 << 17]byte + var x15972 [1 << 17]byte + var x15973 [1 << 17]byte + var x15974 [1 << 17]byte + var x15975 [1 << 17]byte + var x15976 [1 << 17]byte + var x15977 [1 << 17]byte + var x15978 [1 << 17]byte + var x15979 [1 << 17]byte + var x15980 [1 << 17]byte + var x15981 [1 << 17]byte + var x15982 [1 << 17]byte + var x15983 [1 << 17]byte + var x15984 [1 << 17]byte + var x15985 [1 << 17]byte + var x15986 [1 << 17]byte + var x15987 [1 << 17]byte + var x15988 [1 << 17]byte + var x15989 [1 << 17]byte + var x15990 [1 << 17]byte + var x15991 [1 << 17]byte + var x15992 [1 << 17]byte + var x15993 [1 << 17]byte + var x15994 [1 << 17]byte + var x15995 [1 << 17]byte + var x15996 [1 << 17]byte + var x15997 [1 << 17]byte + var x15998 [1 << 17]byte + var x15999 [1 << 17]byte + var x16000 [1 << 17]byte + var x16001 [1 << 17]byte + var x16002 [1 << 17]byte + var x16003 [1 << 17]byte + var x16004 [1 << 17]byte + var x16005 [1 << 17]byte + var x16006 [1 << 17]byte + var x16007 [1 << 17]byte + var x16008 [1 << 17]byte + var x16009 [1 << 17]byte + var x16010 [1 << 17]byte + var x16011 [1 << 17]byte + var x16012 [1 << 17]byte + var x16013 [1 << 17]byte + var x16014 [1 << 17]byte + var x16015 [1 << 17]byte + var x16016 [1 << 17]byte + var x16017 [1 << 17]byte + var x16018 [1 << 17]byte + var x16019 [1 << 17]byte + var x16020 [1 << 17]byte + var x16021 [1 << 17]byte + var x16022 [1 << 17]byte + var x16023 [1 << 17]byte + var x16024 [1 << 17]byte + var x16025 [1 << 17]byte + var x16026 [1 << 17]byte + var x16027 [1 << 17]byte + var x16028 [1 << 17]byte + var x16029 [1 << 17]byte + var x16030 [1 << 17]byte + var x16031 [1 << 17]byte + var x16032 [1 << 17]byte + var x16033 [1 << 17]byte + var x16034 [1 << 17]byte + var x16035 [1 << 17]byte + var x16036 [1 << 17]byte + var x16037 [1 << 17]byte + var x16038 [1 << 17]byte + var x16039 [1 << 17]byte + var x16040 [1 << 17]byte + var x16041 [1 << 17]byte + var x16042 [1 << 17]byte + var x16043 [1 << 17]byte + var x16044 [1 << 17]byte + var x16045 [1 << 17]byte + var x16046 [1 << 17]byte + var x16047 [1 << 17]byte + var x16048 [1 << 17]byte + var x16049 [1 << 17]byte + var x16050 [1 << 17]byte + var x16051 [1 << 17]byte + var x16052 [1 << 17]byte + var x16053 [1 << 17]byte + var x16054 [1 << 17]byte + var x16055 [1 << 17]byte + var x16056 [1 << 17]byte + var x16057 [1 << 17]byte + var x16058 [1 << 17]byte + var x16059 [1 << 17]byte + var x16060 [1 << 17]byte + var x16061 [1 << 17]byte + var x16062 [1 << 17]byte + var x16063 [1 << 17]byte + var x16064 [1 << 17]byte + var x16065 [1 << 17]byte + var x16066 [1 << 17]byte + var x16067 [1 << 17]byte + var x16068 [1 << 17]byte + var x16069 [1 << 17]byte + var x16070 [1 << 17]byte + var x16071 [1 << 17]byte + var x16072 [1 << 17]byte + var x16073 [1 << 17]byte + var x16074 [1 << 17]byte + var x16075 [1 << 17]byte + var x16076 [1 << 17]byte + var x16077 [1 << 17]byte + var x16078 [1 << 17]byte + var x16079 [1 << 17]byte + var x16080 [1 << 17]byte + var x16081 [1 << 17]byte + var x16082 [1 << 17]byte + var x16083 [1 << 17]byte + var x16084 [1 << 17]byte + var x16085 [1 << 17]byte + var x16086 [1 << 17]byte + var x16087 [1 << 17]byte + var x16088 [1 << 17]byte + var x16089 [1 << 17]byte + var x16090 [1 << 17]byte + var x16091 [1 << 17]byte + var x16092 [1 << 17]byte + var x16093 [1 << 17]byte + var x16094 [1 << 17]byte + var x16095 [1 << 17]byte + var x16096 [1 << 17]byte + var x16097 [1 << 17]byte + var x16098 [1 << 17]byte + var x16099 [1 << 17]byte + var x16100 [1 << 17]byte + var x16101 [1 << 17]byte + var x16102 [1 << 17]byte + var x16103 [1 << 17]byte + var x16104 [1 << 17]byte + var x16105 [1 << 17]byte + var x16106 [1 << 17]byte + var x16107 [1 << 17]byte + var x16108 [1 << 17]byte + var x16109 [1 << 17]byte + var x16110 [1 << 17]byte + var x16111 [1 << 17]byte + var x16112 [1 << 17]byte + var x16113 [1 << 17]byte + var x16114 [1 << 17]byte + var x16115 [1 << 17]byte + var x16116 [1 << 17]byte + var x16117 [1 << 17]byte + var x16118 [1 << 17]byte + var x16119 [1 << 17]byte + var x16120 [1 << 17]byte + var x16121 [1 << 17]byte + var x16122 [1 << 17]byte + var x16123 [1 << 17]byte + var x16124 [1 << 17]byte + var x16125 [1 << 17]byte + var x16126 [1 << 17]byte + var x16127 [1 << 17]byte + var x16128 [1 << 17]byte + var x16129 [1 << 17]byte + var x16130 [1 << 17]byte + var x16131 [1 << 17]byte + var x16132 [1 << 17]byte + var x16133 [1 << 17]byte + var x16134 [1 << 17]byte + var x16135 [1 << 17]byte + var x16136 [1 << 17]byte + var x16137 [1 << 17]byte + var x16138 [1 << 17]byte + var x16139 [1 << 17]byte + var x16140 [1 << 17]byte + var x16141 [1 << 17]byte + var x16142 [1 << 17]byte + var x16143 [1 << 17]byte + var x16144 [1 << 17]byte + var x16145 [1 << 17]byte + var x16146 [1 << 17]byte + var x16147 [1 << 17]byte + var x16148 [1 << 17]byte + var x16149 [1 << 17]byte + var x16150 [1 << 17]byte + var x16151 [1 << 17]byte + var x16152 [1 << 17]byte + var x16153 [1 << 17]byte + var x16154 [1 << 17]byte + var x16155 [1 << 17]byte + var x16156 [1 << 17]byte + var x16157 [1 << 17]byte + var x16158 [1 << 17]byte + var x16159 [1 << 17]byte + var x16160 [1 << 17]byte + var x16161 [1 << 17]byte + var x16162 [1 << 17]byte + var x16163 [1 << 17]byte + var x16164 [1 << 17]byte + var x16165 [1 << 17]byte + var x16166 [1 << 17]byte + var x16167 [1 << 17]byte + var x16168 [1 << 17]byte + var x16169 [1 << 17]byte + var x16170 [1 << 17]byte + var x16171 [1 << 17]byte + var x16172 [1 << 17]byte + var x16173 [1 << 17]byte + var x16174 [1 << 17]byte + var x16175 [1 << 17]byte + var x16176 [1 << 17]byte + var x16177 [1 << 17]byte + var x16178 [1 << 17]byte + var x16179 [1 << 17]byte + var x16180 [1 << 17]byte + var x16181 [1 << 17]byte + var x16182 [1 << 17]byte + var x16183 [1 << 17]byte + var x16184 [1 << 17]byte + var x16185 [1 << 17]byte + var x16186 [1 << 17]byte + var x16187 [1 << 17]byte + var x16188 [1 << 17]byte + var x16189 [1 << 17]byte + var x16190 [1 << 17]byte + var x16191 [1 << 17]byte + var x16192 [1 << 17]byte + var x16193 [1 << 17]byte + var x16194 [1 << 17]byte + var x16195 [1 << 17]byte + var x16196 [1 << 17]byte + var x16197 [1 << 17]byte + var x16198 [1 << 17]byte + var x16199 [1 << 17]byte + var x16200 [1 << 17]byte + var x16201 [1 << 17]byte + var x16202 [1 << 17]byte + var x16203 [1 << 17]byte + var x16204 [1 << 17]byte + var x16205 [1 << 17]byte + var x16206 [1 << 17]byte + var x16207 [1 << 17]byte + var x16208 [1 << 17]byte + var x16209 [1 << 17]byte + var x16210 [1 << 17]byte + var x16211 [1 << 17]byte + var x16212 [1 << 17]byte + var x16213 [1 << 17]byte + var x16214 [1 << 17]byte + var x16215 [1 << 17]byte + var x16216 [1 << 17]byte + var x16217 [1 << 17]byte + var x16218 [1 << 17]byte + var x16219 [1 << 17]byte + var x16220 [1 << 17]byte + var x16221 [1 << 17]byte + var x16222 [1 << 17]byte + var x16223 [1 << 17]byte + var x16224 [1 << 17]byte + var x16225 [1 << 17]byte + var x16226 [1 << 17]byte + var x16227 [1 << 17]byte + var x16228 [1 << 17]byte + var x16229 [1 << 17]byte + var x16230 [1 << 17]byte + var x16231 [1 << 17]byte + var x16232 [1 << 17]byte + var x16233 [1 << 17]byte + var x16234 [1 << 17]byte + var x16235 [1 << 17]byte + var x16236 [1 << 17]byte + var x16237 [1 << 17]byte + var x16238 [1 << 17]byte + var x16239 [1 << 17]byte + var x16240 [1 << 17]byte + var x16241 [1 << 17]byte + var x16242 [1 << 17]byte + var x16243 [1 << 17]byte + var x16244 [1 << 17]byte + var x16245 [1 << 17]byte + var x16246 [1 << 17]byte + var x16247 [1 << 17]byte + var x16248 [1 << 17]byte + var x16249 [1 << 17]byte + var x16250 [1 << 17]byte + var x16251 [1 << 17]byte + var x16252 [1 << 17]byte + var x16253 [1 << 17]byte + var x16254 [1 << 17]byte + var x16255 [1 << 17]byte + var x16256 [1 << 17]byte + var x16257 [1 << 17]byte + var x16258 [1 << 17]byte + var x16259 [1 << 17]byte + var x16260 [1 << 17]byte + var x16261 [1 << 17]byte + var x16262 [1 << 17]byte + var x16263 [1 << 17]byte + var x16264 [1 << 17]byte + var x16265 [1 << 17]byte + var x16266 [1 << 17]byte + var x16267 [1 << 17]byte + var x16268 [1 << 17]byte + var x16269 [1 << 17]byte + var x16270 [1 << 17]byte + var x16271 [1 << 17]byte + var x16272 [1 << 17]byte + var x16273 [1 << 17]byte + var x16274 [1 << 17]byte + var x16275 [1 << 17]byte + var x16276 [1 << 17]byte + var x16277 [1 << 17]byte + var x16278 [1 << 17]byte + var x16279 [1 << 17]byte + var x16280 [1 << 17]byte + var x16281 [1 << 17]byte + var x16282 [1 << 17]byte + var x16283 [1 << 17]byte + var x16284 [1 << 17]byte + var x16285 [1 << 17]byte + var x16286 [1 << 17]byte + var x16287 [1 << 17]byte + var x16288 [1 << 17]byte + var x16289 [1 << 17]byte + var x16290 [1 << 17]byte + var x16291 [1 << 17]byte + var x16292 [1 << 17]byte + var x16293 [1 << 17]byte + var x16294 [1 << 17]byte + var x16295 [1 << 17]byte + var x16296 [1 << 17]byte + var x16297 [1 << 17]byte + var x16298 [1 << 17]byte + var x16299 [1 << 17]byte + var x16300 [1 << 17]byte + var x16301 [1 << 17]byte + var x16302 [1 << 17]byte + var x16303 [1 << 17]byte + var x16304 [1 << 17]byte + var x16305 [1 << 17]byte + var x16306 [1 << 17]byte + var x16307 [1 << 17]byte + var x16308 [1 << 17]byte + var x16309 [1 << 17]byte + var x16310 [1 << 17]byte + var x16311 [1 << 17]byte + var x16312 [1 << 17]byte + var x16313 [1 << 17]byte + var x16314 [1 << 17]byte + var x16315 [1 << 17]byte + var x16316 [1 << 17]byte + var x16317 [1 << 17]byte + var x16318 [1 << 17]byte + var x16319 [1 << 17]byte + var x16320 [1 << 17]byte + var x16321 [1 << 17]byte + var x16322 [1 << 17]byte + var x16323 [1 << 17]byte + var x16324 [1 << 17]byte + var x16325 [1 << 17]byte + var x16326 [1 << 17]byte + var x16327 [1 << 17]byte + var x16328 [1 << 17]byte + var x16329 [1 << 17]byte + var x16330 [1 << 17]byte + var x16331 [1 << 17]byte + var x16332 [1 << 17]byte + var x16333 [1 << 17]byte + var x16334 [1 << 17]byte + var x16335 [1 << 17]byte + var x16336 [1 << 17]byte + var x16337 [1 << 17]byte + var x16338 [1 << 17]byte + var x16339 [1 << 17]byte + var x16340 [1 << 17]byte + var x16341 [1 << 17]byte + var x16342 [1 << 17]byte + var x16343 [1 << 17]byte + var x16344 [1 << 17]byte + var x16345 [1 << 17]byte + var x16346 [1 << 17]byte + var x16347 [1 << 17]byte + var x16348 [1 << 17]byte + var x16349 [1 << 17]byte + var x16350 [1 << 17]byte + var x16351 [1 << 17]byte + var x16352 [1 << 17]byte + var x16353 [1 << 17]byte + var x16354 [1 << 17]byte + var x16355 [1 << 17]byte + var x16356 [1 << 17]byte + var x16357 [1 << 17]byte + var x16358 [1 << 17]byte + var x16359 [1 << 17]byte + var x16360 [1 << 17]byte + var x16361 [1 << 17]byte + var x16362 [1 << 17]byte + var x16363 [1 << 17]byte + var x16364 [1 << 17]byte + var x16365 [1 << 17]byte + var x16366 [1 << 17]byte + var x16367 [1 << 17]byte + var x16368 [1 << 17]byte + var x16369 [1 << 17]byte + var x16370 [1 << 17]byte + var x16371 [1 << 17]byte + var x16372 [1 << 17]byte + var x16373 [1 << 17]byte + var x16374 [1 << 17]byte + var x16375 [1 << 17]byte + var x16376 [1 << 17]byte + var x16377 [1 << 17]byte + var x16378 [1 << 17]byte + var x16379 [1 << 17]byte + var x16380 [1 << 17]byte + var x16381 [1 << 17]byte + var x16382 [1 << 17]byte + var x16383 [1 << 17]byte + var x16384 [1 << 17]byte + var x16385 [1 << 17]byte + var x16386 [1 << 17]byte + var x16387 [1 << 17]byte + var x16388 [1 << 17]byte + var x16389 [1 << 17]byte + var x16390 [1 << 17]byte + var x16391 [1 << 17]byte + var x16392 [1 << 17]byte + var x16393 [1 << 17]byte + var x16394 [1 << 17]byte + var x16395 [1 << 17]byte + var x16396 [1 << 17]byte + var x16397 [1 << 17]byte + var x16398 [1 << 17]byte + var x16399 [1 << 17]byte + var x16400 [1 << 17]byte + var x16401 [1 << 17]byte + var x16402 [1 << 17]byte + var x16403 [1 << 17]byte + var x16404 [1 << 17]byte + var x16405 [1 << 17]byte + var x16406 [1 << 17]byte + var x16407 [1 << 17]byte + var x16408 [1 << 17]byte + var x16409 [1 << 17]byte + var x16410 [1 << 17]byte + var x16411 [1 << 17]byte + var x16412 [1 << 17]byte + var x16413 [1 << 17]byte + var x16414 [1 << 17]byte + var x16415 [1 << 17]byte + var x16416 [1 << 17]byte + var x16417 [1 << 17]byte + var x16418 [1 << 17]byte + var x16419 [1 << 17]byte + var x16420 [1 << 17]byte + var x16421 [1 << 17]byte + var x16422 [1 << 17]byte + var x16423 [1 << 17]byte + var x16424 [1 << 17]byte + var x16425 [1 << 17]byte + var x16426 [1 << 17]byte + var x16427 [1 << 17]byte + var x16428 [1 << 17]byte + var x16429 [1 << 17]byte + var x16430 [1 << 17]byte + var x16431 [1 << 17]byte + var x16432 [1 << 17]byte + var x16433 [1 << 17]byte + var x16434 [1 << 17]byte + var x16435 [1 << 17]byte + var x16436 [1 << 17]byte + var x16437 [1 << 17]byte + var x16438 [1 << 17]byte + var x16439 [1 << 17]byte + var x16440 [1 << 17]byte + var x16441 [1 << 17]byte + var x16442 [1 << 17]byte + var x16443 [1 << 17]byte + var x16444 [1 << 17]byte + var x16445 [1 << 17]byte + var x16446 [1 << 17]byte + var x16447 [1 << 17]byte + var x16448 [1 << 17]byte + var x16449 [1 << 17]byte + var x16450 [1 << 17]byte + var x16451 [1 << 17]byte + var x16452 [1 << 17]byte + var x16453 [1 << 17]byte + var x16454 [1 << 17]byte + var x16455 [1 << 17]byte + var x16456 [1 << 17]byte + var x16457 [1 << 17]byte + var x16458 [1 << 17]byte + var x16459 [1 << 17]byte + var x16460 [1 << 17]byte + var x16461 [1 << 17]byte + var x16462 [1 << 17]byte + var x16463 [1 << 17]byte + var x16464 [1 << 17]byte + var x16465 [1 << 17]byte + var x16466 [1 << 17]byte + var x16467 [1 << 17]byte + var x16468 [1 << 17]byte + var x16469 [1 << 17]byte + var x16470 [1 << 17]byte + var x16471 [1 << 17]byte + var x16472 [1 << 17]byte + var x16473 [1 << 17]byte + var x16474 [1 << 17]byte + var x16475 [1 << 17]byte + var x16476 [1 << 17]byte + var x16477 [1 << 17]byte + var x16478 [1 << 17]byte + var x16479 [1 << 17]byte + var x16480 [1 << 17]byte + z1 = x1 + z2 = x2 + z3 = x3 + z4 = x4 + z5 = x5 + z6 = x6 + z7 = x7 + z8 = x8 + z9 = x9 + z10 = x10 + z11 = x11 + z12 = x12 + z13 = x13 + z14 = x14 + z15 = x15 + z16 = x16 + z17 = x17 + z18 = x18 + z19 = x19 + z20 = x20 + z21 = x21 + z22 = x22 + z23 = x23 + z24 = x24 + z25 = x25 + z26 = x26 + z27 = x27 + z28 = x28 + z29 = x29 + z30 = x30 + z31 = x31 + z32 = x32 + z33 = x33 + z34 = x34 + z35 = x35 + z36 = x36 + z37 = x37 + z38 = x38 + z39 = x39 + z40 = x40 + z41 = x41 + z42 = x42 + z43 = x43 + z44 = x44 + z45 = x45 + z46 = x46 + z47 = x47 + z48 = x48 + z49 = x49 + z50 = x50 + z51 = x51 + z52 = x52 + z53 = x53 + z54 = x54 + z55 = x55 + z56 = x56 + z57 = x57 + z58 = x58 + z59 = x59 + z60 = x60 + z61 = x61 + z62 = x62 + z63 = x63 + z64 = x64 + z65 = x65 + z66 = x66 + z67 = x67 + z68 = x68 + z69 = x69 + z70 = x70 + z71 = x71 + z72 = x72 + z73 = x73 + z74 = x74 + z75 = x75 + z76 = x76 + z77 = x77 + z78 = x78 + z79 = x79 + z80 = x80 + z81 = x81 + z82 = x82 + z83 = x83 + z84 = x84 + z85 = x85 + z86 = x86 + z87 = x87 + z88 = x88 + z89 = x89 + z90 = x90 + z91 = x91 + z92 = x92 + z93 = x93 + z94 = x94 + z95 = x95 + z96 = x96 + z97 = x97 + z98 = x98 + z99 = x99 + z100 = x100 + z101 = x101 + z102 = x102 + z103 = x103 + z104 = x104 + z105 = x105 + z106 = x106 + z107 = x107 + z108 = x108 + z109 = x109 + z110 = x110 + z111 = x111 + z112 = x112 + z113 = x113 + z114 = x114 + z115 = x115 + z116 = x116 + z117 = x117 + z118 = x118 + z119 = x119 + z120 = x120 + z121 = x121 + z122 = x122 + z123 = x123 + z124 = x124 + z125 = x125 + z126 = x126 + z127 = x127 + z128 = x128 + z129 = x129 + z130 = x130 + z131 = x131 + z132 = x132 + z133 = x133 + z134 = x134 + z135 = x135 + z136 = x136 + z137 = x137 + z138 = x138 + z139 = x139 + z140 = x140 + z141 = x141 + z142 = x142 + z143 = x143 + z144 = x144 + z145 = x145 + z146 = x146 + z147 = x147 + z148 = x148 + z149 = x149 + z150 = x150 + z151 = x151 + z152 = x152 + z153 = x153 + z154 = x154 + z155 = x155 + z156 = x156 + z157 = x157 + z158 = x158 + z159 = x159 + z160 = x160 + z161 = x161 + z162 = x162 + z163 = x163 + z164 = x164 + z165 = x165 + z166 = x166 + z167 = x167 + z168 = x168 + z169 = x169 + z170 = x170 + z171 = x171 + z172 = x172 + z173 = x173 + z174 = x174 + z175 = x175 + z176 = x176 + z177 = x177 + z178 = x178 + z179 = x179 + z180 = x180 + z181 = x181 + z182 = x182 + z183 = x183 + z184 = x184 + z185 = x185 + z186 = x186 + z187 = x187 + z188 = x188 + z189 = x189 + z190 = x190 + z191 = x191 + z192 = x192 + z193 = x193 + z194 = x194 + z195 = x195 + z196 = x196 + z197 = x197 + z198 = x198 + z199 = x199 + z200 = x200 + z201 = x201 + z202 = x202 + z203 = x203 + z204 = x204 + z205 = x205 + z206 = x206 + z207 = x207 + z208 = x208 + z209 = x209 + z210 = x210 + z211 = x211 + z212 = x212 + z213 = x213 + z214 = x214 + z215 = x215 + z216 = x216 + z217 = x217 + z218 = x218 + z219 = x219 + z220 = x220 + z221 = x221 + z222 = x222 + z223 = x223 + z224 = x224 + z225 = x225 + z226 = x226 + z227 = x227 + z228 = x228 + z229 = x229 + z230 = x230 + z231 = x231 + z232 = x232 + z233 = x233 + z234 = x234 + z235 = x235 + z236 = x236 + z237 = x237 + z238 = x238 + z239 = x239 + z240 = x240 + z241 = x241 + z242 = x242 + z243 = x243 + z244 = x244 + z245 = x245 + z246 = x246 + z247 = x247 + z248 = x248 + z249 = x249 + z250 = x250 + z251 = x251 + z252 = x252 + z253 = x253 + z254 = x254 + z255 = x255 + z256 = x256 + z257 = x257 + z258 = x258 + z259 = x259 + z260 = x260 + z261 = x261 + z262 = x262 + z263 = x263 + z264 = x264 + z265 = x265 + z266 = x266 + z267 = x267 + z268 = x268 + z269 = x269 + z270 = x270 + z271 = x271 + z272 = x272 + z273 = x273 + z274 = x274 + z275 = x275 + z276 = x276 + z277 = x277 + z278 = x278 + z279 = x279 + z280 = x280 + z281 = x281 + z282 = x282 + z283 = x283 + z284 = x284 + z285 = x285 + z286 = x286 + z287 = x287 + z288 = x288 + z289 = x289 + z290 = x290 + z291 = x291 + z292 = x292 + z293 = x293 + z294 = x294 + z295 = x295 + z296 = x296 + z297 = x297 + z298 = x298 + z299 = x299 + z300 = x300 + z301 = x301 + z302 = x302 + z303 = x303 + z304 = x304 + z305 = x305 + z306 = x306 + z307 = x307 + z308 = x308 + z309 = x309 + z310 = x310 + z311 = x311 + z312 = x312 + z313 = x313 + z314 = x314 + z315 = x315 + z316 = x316 + z317 = x317 + z318 = x318 + z319 = x319 + z320 = x320 + z321 = x321 + z322 = x322 + z323 = x323 + z324 = x324 + z325 = x325 + z326 = x326 + z327 = x327 + z328 = x328 + z329 = x329 + z330 = x330 + z331 = x331 + z332 = x332 + z333 = x333 + z334 = x334 + z335 = x335 + z336 = x336 + z337 = x337 + z338 = x338 + z339 = x339 + z340 = x340 + z341 = x341 + z342 = x342 + z343 = x343 + z344 = x344 + z345 = x345 + z346 = x346 + z347 = x347 + z348 = x348 + z349 = x349 + z350 = x350 + z351 = x351 + z352 = x352 + z353 = x353 + z354 = x354 + z355 = x355 + z356 = x356 + z357 = x357 + z358 = x358 + z359 = x359 + z360 = x360 + z361 = x361 + z362 = x362 + z363 = x363 + z364 = x364 + z365 = x365 + z366 = x366 + z367 = x367 + z368 = x368 + z369 = x369 + z370 = x370 + z371 = x371 + z372 = x372 + z373 = x373 + z374 = x374 + z375 = x375 + z376 = x376 + z377 = x377 + z378 = x378 + z379 = x379 + z380 = x380 + z381 = x381 + z382 = x382 + z383 = x383 + z384 = x384 + z385 = x385 + z386 = x386 + z387 = x387 + z388 = x388 + z389 = x389 + z390 = x390 + z391 = x391 + z392 = x392 + z393 = x393 + z394 = x394 + z395 = x395 + z396 = x396 + z397 = x397 + z398 = x398 + z399 = x399 + z400 = x400 + z401 = x401 + z402 = x402 + z403 = x403 + z404 = x404 + z405 = x405 + z406 = x406 + z407 = x407 + z408 = x408 + z409 = x409 + z410 = x410 + z411 = x411 + z412 = x412 + z413 = x413 + z414 = x414 + z415 = x415 + z416 = x416 + z417 = x417 + z418 = x418 + z419 = x419 + z420 = x420 + z421 = x421 + z422 = x422 + z423 = x423 + z424 = x424 + z425 = x425 + z426 = x426 + z427 = x427 + z428 = x428 + z429 = x429 + z430 = x430 + z431 = x431 + z432 = x432 + z433 = x433 + z434 = x434 + z435 = x435 + z436 = x436 + z437 = x437 + z438 = x438 + z439 = x439 + z440 = x440 + z441 = x441 + z442 = x442 + z443 = x443 + z444 = x444 + z445 = x445 + z446 = x446 + z447 = x447 + z448 = x448 + z449 = x449 + z450 = x450 + z451 = x451 + z452 = x452 + z453 = x453 + z454 = x454 + z455 = x455 + z456 = x456 + z457 = x457 + z458 = x458 + z459 = x459 + z460 = x460 + z461 = x461 + z462 = x462 + z463 = x463 + z464 = x464 + z465 = x465 + z466 = x466 + z467 = x467 + z468 = x468 + z469 = x469 + z470 = x470 + z471 = x471 + z472 = x472 + z473 = x473 + z474 = x474 + z475 = x475 + z476 = x476 + z477 = x477 + z478 = x478 + z479 = x479 + z480 = x480 + z481 = x481 + z482 = x482 + z483 = x483 + z484 = x484 + z485 = x485 + z486 = x486 + z487 = x487 + z488 = x488 + z489 = x489 + z490 = x490 + z491 = x491 + z492 = x492 + z493 = x493 + z494 = x494 + z495 = x495 + z496 = x496 + z497 = x497 + z498 = x498 + z499 = x499 + z500 = x500 + z501 = x501 + z502 = x502 + z503 = x503 + z504 = x504 + z505 = x505 + z506 = x506 + z507 = x507 + z508 = x508 + z509 = x509 + z510 = x510 + z511 = x511 + z512 = x512 + z513 = x513 + z514 = x514 + z515 = x515 + z516 = x516 + z517 = x517 + z518 = x518 + z519 = x519 + z520 = x520 + z521 = x521 + z522 = x522 + z523 = x523 + z524 = x524 + z525 = x525 + z526 = x526 + z527 = x527 + z528 = x528 + z529 = x529 + z530 = x530 + z531 = x531 + z532 = x532 + z533 = x533 + z534 = x534 + z535 = x535 + z536 = x536 + z537 = x537 + z538 = x538 + z539 = x539 + z540 = x540 + z541 = x541 + z542 = x542 + z543 = x543 + z544 = x544 + z545 = x545 + z546 = x546 + z547 = x547 + z548 = x548 + z549 = x549 + z550 = x550 + z551 = x551 + z552 = x552 + z553 = x553 + z554 = x554 + z555 = x555 + z556 = x556 + z557 = x557 + z558 = x558 + z559 = x559 + z560 = x560 + z561 = x561 + z562 = x562 + z563 = x563 + z564 = x564 + z565 = x565 + z566 = x566 + z567 = x567 + z568 = x568 + z569 = x569 + z570 = x570 + z571 = x571 + z572 = x572 + z573 = x573 + z574 = x574 + z575 = x575 + z576 = x576 + z577 = x577 + z578 = x578 + z579 = x579 + z580 = x580 + z581 = x581 + z582 = x582 + z583 = x583 + z584 = x584 + z585 = x585 + z586 = x586 + z587 = x587 + z588 = x588 + z589 = x589 + z590 = x590 + z591 = x591 + z592 = x592 + z593 = x593 + z594 = x594 + z595 = x595 + z596 = x596 + z597 = x597 + z598 = x598 + z599 = x599 + z600 = x600 + z601 = x601 + z602 = x602 + z603 = x603 + z604 = x604 + z605 = x605 + z606 = x606 + z607 = x607 + z608 = x608 + z609 = x609 + z610 = x610 + z611 = x611 + z612 = x612 + z613 = x613 + z614 = x614 + z615 = x615 + z616 = x616 + z617 = x617 + z618 = x618 + z619 = x619 + z620 = x620 + z621 = x621 + z622 = x622 + z623 = x623 + z624 = x624 + z625 = x625 + z626 = x626 + z627 = x627 + z628 = x628 + z629 = x629 + z630 = x630 + z631 = x631 + z632 = x632 + z633 = x633 + z634 = x634 + z635 = x635 + z636 = x636 + z637 = x637 + z638 = x638 + z639 = x639 + z640 = x640 + z641 = x641 + z642 = x642 + z643 = x643 + z644 = x644 + z645 = x645 + z646 = x646 + z647 = x647 + z648 = x648 + z649 = x649 + z650 = x650 + z651 = x651 + z652 = x652 + z653 = x653 + z654 = x654 + z655 = x655 + z656 = x656 + z657 = x657 + z658 = x658 + z659 = x659 + z660 = x660 + z661 = x661 + z662 = x662 + z663 = x663 + z664 = x664 + z665 = x665 + z666 = x666 + z667 = x667 + z668 = x668 + z669 = x669 + z670 = x670 + z671 = x671 + z672 = x672 + z673 = x673 + z674 = x674 + z675 = x675 + z676 = x676 + z677 = x677 + z678 = x678 + z679 = x679 + z680 = x680 + z681 = x681 + z682 = x682 + z683 = x683 + z684 = x684 + z685 = x685 + z686 = x686 + z687 = x687 + z688 = x688 + z689 = x689 + z690 = x690 + z691 = x691 + z692 = x692 + z693 = x693 + z694 = x694 + z695 = x695 + z696 = x696 + z697 = x697 + z698 = x698 + z699 = x699 + z700 = x700 + z701 = x701 + z702 = x702 + z703 = x703 + z704 = x704 + z705 = x705 + z706 = x706 + z707 = x707 + z708 = x708 + z709 = x709 + z710 = x710 + z711 = x711 + z712 = x712 + z713 = x713 + z714 = x714 + z715 = x715 + z716 = x716 + z717 = x717 + z718 = x718 + z719 = x719 + z720 = x720 + z721 = x721 + z722 = x722 + z723 = x723 + z724 = x724 + z725 = x725 + z726 = x726 + z727 = x727 + z728 = x728 + z729 = x729 + z730 = x730 + z731 = x731 + z732 = x732 + z733 = x733 + z734 = x734 + z735 = x735 + z736 = x736 + z737 = x737 + z738 = x738 + z739 = x739 + z740 = x740 + z741 = x741 + z742 = x742 + z743 = x743 + z744 = x744 + z745 = x745 + z746 = x746 + z747 = x747 + z748 = x748 + z749 = x749 + z750 = x750 + z751 = x751 + z752 = x752 + z753 = x753 + z754 = x754 + z755 = x755 + z756 = x756 + z757 = x757 + z758 = x758 + z759 = x759 + z760 = x760 + z761 = x761 + z762 = x762 + z763 = x763 + z764 = x764 + z765 = x765 + z766 = x766 + z767 = x767 + z768 = x768 + z769 = x769 + z770 = x770 + z771 = x771 + z772 = x772 + z773 = x773 + z774 = x774 + z775 = x775 + z776 = x776 + z777 = x777 + z778 = x778 + z779 = x779 + z780 = x780 + z781 = x781 + z782 = x782 + z783 = x783 + z784 = x784 + z785 = x785 + z786 = x786 + z787 = x787 + z788 = x788 + z789 = x789 + z790 = x790 + z791 = x791 + z792 = x792 + z793 = x793 + z794 = x794 + z795 = x795 + z796 = x796 + z797 = x797 + z798 = x798 + z799 = x799 + z800 = x800 + z801 = x801 + z802 = x802 + z803 = x803 + z804 = x804 + z805 = x805 + z806 = x806 + z807 = x807 + z808 = x808 + z809 = x809 + z810 = x810 + z811 = x811 + z812 = x812 + z813 = x813 + z814 = x814 + z815 = x815 + z816 = x816 + z817 = x817 + z818 = x818 + z819 = x819 + z820 = x820 + z821 = x821 + z822 = x822 + z823 = x823 + z824 = x824 + z825 = x825 + z826 = x826 + z827 = x827 + z828 = x828 + z829 = x829 + z830 = x830 + z831 = x831 + z832 = x832 + z833 = x833 + z834 = x834 + z835 = x835 + z836 = x836 + z837 = x837 + z838 = x838 + z839 = x839 + z840 = x840 + z841 = x841 + z842 = x842 + z843 = x843 + z844 = x844 + z845 = x845 + z846 = x846 + z847 = x847 + z848 = x848 + z849 = x849 + z850 = x850 + z851 = x851 + z852 = x852 + z853 = x853 + z854 = x854 + z855 = x855 + z856 = x856 + z857 = x857 + z858 = x858 + z859 = x859 + z860 = x860 + z861 = x861 + z862 = x862 + z863 = x863 + z864 = x864 + z865 = x865 + z866 = x866 + z867 = x867 + z868 = x868 + z869 = x869 + z870 = x870 + z871 = x871 + z872 = x872 + z873 = x873 + z874 = x874 + z875 = x875 + z876 = x876 + z877 = x877 + z878 = x878 + z879 = x879 + z880 = x880 + z881 = x881 + z882 = x882 + z883 = x883 + z884 = x884 + z885 = x885 + z886 = x886 + z887 = x887 + z888 = x888 + z889 = x889 + z890 = x890 + z891 = x891 + z892 = x892 + z893 = x893 + z894 = x894 + z895 = x895 + z896 = x896 + z897 = x897 + z898 = x898 + z899 = x899 + z900 = x900 + z901 = x901 + z902 = x902 + z903 = x903 + z904 = x904 + z905 = x905 + z906 = x906 + z907 = x907 + z908 = x908 + z909 = x909 + z910 = x910 + z911 = x911 + z912 = x912 + z913 = x913 + z914 = x914 + z915 = x915 + z916 = x916 + z917 = x917 + z918 = x918 + z919 = x919 + z920 = x920 + z921 = x921 + z922 = x922 + z923 = x923 + z924 = x924 + z925 = x925 + z926 = x926 + z927 = x927 + z928 = x928 + z929 = x929 + z930 = x930 + z931 = x931 + z932 = x932 + z933 = x933 + z934 = x934 + z935 = x935 + z936 = x936 + z937 = x937 + z938 = x938 + z939 = x939 + z940 = x940 + z941 = x941 + z942 = x942 + z943 = x943 + z944 = x944 + z945 = x945 + z946 = x946 + z947 = x947 + z948 = x948 + z949 = x949 + z950 = x950 + z951 = x951 + z952 = x952 + z953 = x953 + z954 = x954 + z955 = x955 + z956 = x956 + z957 = x957 + z958 = x958 + z959 = x959 + z960 = x960 + z961 = x961 + z962 = x962 + z963 = x963 + z964 = x964 + z965 = x965 + z966 = x966 + z967 = x967 + z968 = x968 + z969 = x969 + z970 = x970 + z971 = x971 + z972 = x972 + z973 = x973 + z974 = x974 + z975 = x975 + z976 = x976 + z977 = x977 + z978 = x978 + z979 = x979 + z980 = x980 + z981 = x981 + z982 = x982 + z983 = x983 + z984 = x984 + z985 = x985 + z986 = x986 + z987 = x987 + z988 = x988 + z989 = x989 + z990 = x990 + z991 = x991 + z992 = x992 + z993 = x993 + z994 = x994 + z995 = x995 + z996 = x996 + z997 = x997 + z998 = x998 + z999 = x999 + z1000 = x1000 + z1001 = x1001 + z1002 = x1002 + z1003 = x1003 + z1004 = x1004 + z1005 = x1005 + z1006 = x1006 + z1007 = x1007 + z1008 = x1008 + z1009 = x1009 + z1010 = x1010 + z1011 = x1011 + z1012 = x1012 + z1013 = x1013 + z1014 = x1014 + z1015 = x1015 + z1016 = x1016 + z1017 = x1017 + z1018 = x1018 + z1019 = x1019 + z1020 = x1020 + z1021 = x1021 + z1022 = x1022 + z1023 = x1023 + z1024 = x1024 + z1025 = x1025 + z1026 = x1026 + z1027 = x1027 + z1028 = x1028 + z1029 = x1029 + z1030 = x1030 + z1031 = x1031 + z1032 = x1032 + z1033 = x1033 + z1034 = x1034 + z1035 = x1035 + z1036 = x1036 + z1037 = x1037 + z1038 = x1038 + z1039 = x1039 + z1040 = x1040 + z1041 = x1041 + z1042 = x1042 + z1043 = x1043 + z1044 = x1044 + z1045 = x1045 + z1046 = x1046 + z1047 = x1047 + z1048 = x1048 + z1049 = x1049 + z1050 = x1050 + z1051 = x1051 + z1052 = x1052 + z1053 = x1053 + z1054 = x1054 + z1055 = x1055 + z1056 = x1056 + z1057 = x1057 + z1058 = x1058 + z1059 = x1059 + z1060 = x1060 + z1061 = x1061 + z1062 = x1062 + z1063 = x1063 + z1064 = x1064 + z1065 = x1065 + z1066 = x1066 + z1067 = x1067 + z1068 = x1068 + z1069 = x1069 + z1070 = x1070 + z1071 = x1071 + z1072 = x1072 + z1073 = x1073 + z1074 = x1074 + z1075 = x1075 + z1076 = x1076 + z1077 = x1077 + z1078 = x1078 + z1079 = x1079 + z1080 = x1080 + z1081 = x1081 + z1082 = x1082 + z1083 = x1083 + z1084 = x1084 + z1085 = x1085 + z1086 = x1086 + z1087 = x1087 + z1088 = x1088 + z1089 = x1089 + z1090 = x1090 + z1091 = x1091 + z1092 = x1092 + z1093 = x1093 + z1094 = x1094 + z1095 = x1095 + z1096 = x1096 + z1097 = x1097 + z1098 = x1098 + z1099 = x1099 + z1100 = x1100 + z1101 = x1101 + z1102 = x1102 + z1103 = x1103 + z1104 = x1104 + z1105 = x1105 + z1106 = x1106 + z1107 = x1107 + z1108 = x1108 + z1109 = x1109 + z1110 = x1110 + z1111 = x1111 + z1112 = x1112 + z1113 = x1113 + z1114 = x1114 + z1115 = x1115 + z1116 = x1116 + z1117 = x1117 + z1118 = x1118 + z1119 = x1119 + z1120 = x1120 + z1121 = x1121 + z1122 = x1122 + z1123 = x1123 + z1124 = x1124 + z1125 = x1125 + z1126 = x1126 + z1127 = x1127 + z1128 = x1128 + z1129 = x1129 + z1130 = x1130 + z1131 = x1131 + z1132 = x1132 + z1133 = x1133 + z1134 = x1134 + z1135 = x1135 + z1136 = x1136 + z1137 = x1137 + z1138 = x1138 + z1139 = x1139 + z1140 = x1140 + z1141 = x1141 + z1142 = x1142 + z1143 = x1143 + z1144 = x1144 + z1145 = x1145 + z1146 = x1146 + z1147 = x1147 + z1148 = x1148 + z1149 = x1149 + z1150 = x1150 + z1151 = x1151 + z1152 = x1152 + z1153 = x1153 + z1154 = x1154 + z1155 = x1155 + z1156 = x1156 + z1157 = x1157 + z1158 = x1158 + z1159 = x1159 + z1160 = x1160 + z1161 = x1161 + z1162 = x1162 + z1163 = x1163 + z1164 = x1164 + z1165 = x1165 + z1166 = x1166 + z1167 = x1167 + z1168 = x1168 + z1169 = x1169 + z1170 = x1170 + z1171 = x1171 + z1172 = x1172 + z1173 = x1173 + z1174 = x1174 + z1175 = x1175 + z1176 = x1176 + z1177 = x1177 + z1178 = x1178 + z1179 = x1179 + z1180 = x1180 + z1181 = x1181 + z1182 = x1182 + z1183 = x1183 + z1184 = x1184 + z1185 = x1185 + z1186 = x1186 + z1187 = x1187 + z1188 = x1188 + z1189 = x1189 + z1190 = x1190 + z1191 = x1191 + z1192 = x1192 + z1193 = x1193 + z1194 = x1194 + z1195 = x1195 + z1196 = x1196 + z1197 = x1197 + z1198 = x1198 + z1199 = x1199 + z1200 = x1200 + z1201 = x1201 + z1202 = x1202 + z1203 = x1203 + z1204 = x1204 + z1205 = x1205 + z1206 = x1206 + z1207 = x1207 + z1208 = x1208 + z1209 = x1209 + z1210 = x1210 + z1211 = x1211 + z1212 = x1212 + z1213 = x1213 + z1214 = x1214 + z1215 = x1215 + z1216 = x1216 + z1217 = x1217 + z1218 = x1218 + z1219 = x1219 + z1220 = x1220 + z1221 = x1221 + z1222 = x1222 + z1223 = x1223 + z1224 = x1224 + z1225 = x1225 + z1226 = x1226 + z1227 = x1227 + z1228 = x1228 + z1229 = x1229 + z1230 = x1230 + z1231 = x1231 + z1232 = x1232 + z1233 = x1233 + z1234 = x1234 + z1235 = x1235 + z1236 = x1236 + z1237 = x1237 + z1238 = x1238 + z1239 = x1239 + z1240 = x1240 + z1241 = x1241 + z1242 = x1242 + z1243 = x1243 + z1244 = x1244 + z1245 = x1245 + z1246 = x1246 + z1247 = x1247 + z1248 = x1248 + z1249 = x1249 + z1250 = x1250 + z1251 = x1251 + z1252 = x1252 + z1253 = x1253 + z1254 = x1254 + z1255 = x1255 + z1256 = x1256 + z1257 = x1257 + z1258 = x1258 + z1259 = x1259 + z1260 = x1260 + z1261 = x1261 + z1262 = x1262 + z1263 = x1263 + z1264 = x1264 + z1265 = x1265 + z1266 = x1266 + z1267 = x1267 + z1268 = x1268 + z1269 = x1269 + z1270 = x1270 + z1271 = x1271 + z1272 = x1272 + z1273 = x1273 + z1274 = x1274 + z1275 = x1275 + z1276 = x1276 + z1277 = x1277 + z1278 = x1278 + z1279 = x1279 + z1280 = x1280 + z1281 = x1281 + z1282 = x1282 + z1283 = x1283 + z1284 = x1284 + z1285 = x1285 + z1286 = x1286 + z1287 = x1287 + z1288 = x1288 + z1289 = x1289 + z1290 = x1290 + z1291 = x1291 + z1292 = x1292 + z1293 = x1293 + z1294 = x1294 + z1295 = x1295 + z1296 = x1296 + z1297 = x1297 + z1298 = x1298 + z1299 = x1299 + z1300 = x1300 + z1301 = x1301 + z1302 = x1302 + z1303 = x1303 + z1304 = x1304 + z1305 = x1305 + z1306 = x1306 + z1307 = x1307 + z1308 = x1308 + z1309 = x1309 + z1310 = x1310 + z1311 = x1311 + z1312 = x1312 + z1313 = x1313 + z1314 = x1314 + z1315 = x1315 + z1316 = x1316 + z1317 = x1317 + z1318 = x1318 + z1319 = x1319 + z1320 = x1320 + z1321 = x1321 + z1322 = x1322 + z1323 = x1323 + z1324 = x1324 + z1325 = x1325 + z1326 = x1326 + z1327 = x1327 + z1328 = x1328 + z1329 = x1329 + z1330 = x1330 + z1331 = x1331 + z1332 = x1332 + z1333 = x1333 + z1334 = x1334 + z1335 = x1335 + z1336 = x1336 + z1337 = x1337 + z1338 = x1338 + z1339 = x1339 + z1340 = x1340 + z1341 = x1341 + z1342 = x1342 + z1343 = x1343 + z1344 = x1344 + z1345 = x1345 + z1346 = x1346 + z1347 = x1347 + z1348 = x1348 + z1349 = x1349 + z1350 = x1350 + z1351 = x1351 + z1352 = x1352 + z1353 = x1353 + z1354 = x1354 + z1355 = x1355 + z1356 = x1356 + z1357 = x1357 + z1358 = x1358 + z1359 = x1359 + z1360 = x1360 + z1361 = x1361 + z1362 = x1362 + z1363 = x1363 + z1364 = x1364 + z1365 = x1365 + z1366 = x1366 + z1367 = x1367 + z1368 = x1368 + z1369 = x1369 + z1370 = x1370 + z1371 = x1371 + z1372 = x1372 + z1373 = x1373 + z1374 = x1374 + z1375 = x1375 + z1376 = x1376 + z1377 = x1377 + z1378 = x1378 + z1379 = x1379 + z1380 = x1380 + z1381 = x1381 + z1382 = x1382 + z1383 = x1383 + z1384 = x1384 + z1385 = x1385 + z1386 = x1386 + z1387 = x1387 + z1388 = x1388 + z1389 = x1389 + z1390 = x1390 + z1391 = x1391 + z1392 = x1392 + z1393 = x1393 + z1394 = x1394 + z1395 = x1395 + z1396 = x1396 + z1397 = x1397 + z1398 = x1398 + z1399 = x1399 + z1400 = x1400 + z1401 = x1401 + z1402 = x1402 + z1403 = x1403 + z1404 = x1404 + z1405 = x1405 + z1406 = x1406 + z1407 = x1407 + z1408 = x1408 + z1409 = x1409 + z1410 = x1410 + z1411 = x1411 + z1412 = x1412 + z1413 = x1413 + z1414 = x1414 + z1415 = x1415 + z1416 = x1416 + z1417 = x1417 + z1418 = x1418 + z1419 = x1419 + z1420 = x1420 + z1421 = x1421 + z1422 = x1422 + z1423 = x1423 + z1424 = x1424 + z1425 = x1425 + z1426 = x1426 + z1427 = x1427 + z1428 = x1428 + z1429 = x1429 + z1430 = x1430 + z1431 = x1431 + z1432 = x1432 + z1433 = x1433 + z1434 = x1434 + z1435 = x1435 + z1436 = x1436 + z1437 = x1437 + z1438 = x1438 + z1439 = x1439 + z1440 = x1440 + z1441 = x1441 + z1442 = x1442 + z1443 = x1443 + z1444 = x1444 + z1445 = x1445 + z1446 = x1446 + z1447 = x1447 + z1448 = x1448 + z1449 = x1449 + z1450 = x1450 + z1451 = x1451 + z1452 = x1452 + z1453 = x1453 + z1454 = x1454 + z1455 = x1455 + z1456 = x1456 + z1457 = x1457 + z1458 = x1458 + z1459 = x1459 + z1460 = x1460 + z1461 = x1461 + z1462 = x1462 + z1463 = x1463 + z1464 = x1464 + z1465 = x1465 + z1466 = x1466 + z1467 = x1467 + z1468 = x1468 + z1469 = x1469 + z1470 = x1470 + z1471 = x1471 + z1472 = x1472 + z1473 = x1473 + z1474 = x1474 + z1475 = x1475 + z1476 = x1476 + z1477 = x1477 + z1478 = x1478 + z1479 = x1479 + z1480 = x1480 + z1481 = x1481 + z1482 = x1482 + z1483 = x1483 + z1484 = x1484 + z1485 = x1485 + z1486 = x1486 + z1487 = x1487 + z1488 = x1488 + z1489 = x1489 + z1490 = x1490 + z1491 = x1491 + z1492 = x1492 + z1493 = x1493 + z1494 = x1494 + z1495 = x1495 + z1496 = x1496 + z1497 = x1497 + z1498 = x1498 + z1499 = x1499 + z1500 = x1500 + z1501 = x1501 + z1502 = x1502 + z1503 = x1503 + z1504 = x1504 + z1505 = x1505 + z1506 = x1506 + z1507 = x1507 + z1508 = x1508 + z1509 = x1509 + z1510 = x1510 + z1511 = x1511 + z1512 = x1512 + z1513 = x1513 + z1514 = x1514 + z1515 = x1515 + z1516 = x1516 + z1517 = x1517 + z1518 = x1518 + z1519 = x1519 + z1520 = x1520 + z1521 = x1521 + z1522 = x1522 + z1523 = x1523 + z1524 = x1524 + z1525 = x1525 + z1526 = x1526 + z1527 = x1527 + z1528 = x1528 + z1529 = x1529 + z1530 = x1530 + z1531 = x1531 + z1532 = x1532 + z1533 = x1533 + z1534 = x1534 + z1535 = x1535 + z1536 = x1536 + z1537 = x1537 + z1538 = x1538 + z1539 = x1539 + z1540 = x1540 + z1541 = x1541 + z1542 = x1542 + z1543 = x1543 + z1544 = x1544 + z1545 = x1545 + z1546 = x1546 + z1547 = x1547 + z1548 = x1548 + z1549 = x1549 + z1550 = x1550 + z1551 = x1551 + z1552 = x1552 + z1553 = x1553 + z1554 = x1554 + z1555 = x1555 + z1556 = x1556 + z1557 = x1557 + z1558 = x1558 + z1559 = x1559 + z1560 = x1560 + z1561 = x1561 + z1562 = x1562 + z1563 = x1563 + z1564 = x1564 + z1565 = x1565 + z1566 = x1566 + z1567 = x1567 + z1568 = x1568 + z1569 = x1569 + z1570 = x1570 + z1571 = x1571 + z1572 = x1572 + z1573 = x1573 + z1574 = x1574 + z1575 = x1575 + z1576 = x1576 + z1577 = x1577 + z1578 = x1578 + z1579 = x1579 + z1580 = x1580 + z1581 = x1581 + z1582 = x1582 + z1583 = x1583 + z1584 = x1584 + z1585 = x1585 + z1586 = x1586 + z1587 = x1587 + z1588 = x1588 + z1589 = x1589 + z1590 = x1590 + z1591 = x1591 + z1592 = x1592 + z1593 = x1593 + z1594 = x1594 + z1595 = x1595 + z1596 = x1596 + z1597 = x1597 + z1598 = x1598 + z1599 = x1599 + z1600 = x1600 + z1601 = x1601 + z1602 = x1602 + z1603 = x1603 + z1604 = x1604 + z1605 = x1605 + z1606 = x1606 + z1607 = x1607 + z1608 = x1608 + z1609 = x1609 + z1610 = x1610 + z1611 = x1611 + z1612 = x1612 + z1613 = x1613 + z1614 = x1614 + z1615 = x1615 + z1616 = x1616 + z1617 = x1617 + z1618 = x1618 + z1619 = x1619 + z1620 = x1620 + z1621 = x1621 + z1622 = x1622 + z1623 = x1623 + z1624 = x1624 + z1625 = x1625 + z1626 = x1626 + z1627 = x1627 + z1628 = x1628 + z1629 = x1629 + z1630 = x1630 + z1631 = x1631 + z1632 = x1632 + z1633 = x1633 + z1634 = x1634 + z1635 = x1635 + z1636 = x1636 + z1637 = x1637 + z1638 = x1638 + z1639 = x1639 + z1640 = x1640 + z1641 = x1641 + z1642 = x1642 + z1643 = x1643 + z1644 = x1644 + z1645 = x1645 + z1646 = x1646 + z1647 = x1647 + z1648 = x1648 + z1649 = x1649 + z1650 = x1650 + z1651 = x1651 + z1652 = x1652 + z1653 = x1653 + z1654 = x1654 + z1655 = x1655 + z1656 = x1656 + z1657 = x1657 + z1658 = x1658 + z1659 = x1659 + z1660 = x1660 + z1661 = x1661 + z1662 = x1662 + z1663 = x1663 + z1664 = x1664 + z1665 = x1665 + z1666 = x1666 + z1667 = x1667 + z1668 = x1668 + z1669 = x1669 + z1670 = x1670 + z1671 = x1671 + z1672 = x1672 + z1673 = x1673 + z1674 = x1674 + z1675 = x1675 + z1676 = x1676 + z1677 = x1677 + z1678 = x1678 + z1679 = x1679 + z1680 = x1680 + z1681 = x1681 + z1682 = x1682 + z1683 = x1683 + z1684 = x1684 + z1685 = x1685 + z1686 = x1686 + z1687 = x1687 + z1688 = x1688 + z1689 = x1689 + z1690 = x1690 + z1691 = x1691 + z1692 = x1692 + z1693 = x1693 + z1694 = x1694 + z1695 = x1695 + z1696 = x1696 + z1697 = x1697 + z1698 = x1698 + z1699 = x1699 + z1700 = x1700 + z1701 = x1701 + z1702 = x1702 + z1703 = x1703 + z1704 = x1704 + z1705 = x1705 + z1706 = x1706 + z1707 = x1707 + z1708 = x1708 + z1709 = x1709 + z1710 = x1710 + z1711 = x1711 + z1712 = x1712 + z1713 = x1713 + z1714 = x1714 + z1715 = x1715 + z1716 = x1716 + z1717 = x1717 + z1718 = x1718 + z1719 = x1719 + z1720 = x1720 + z1721 = x1721 + z1722 = x1722 + z1723 = x1723 + z1724 = x1724 + z1725 = x1725 + z1726 = x1726 + z1727 = x1727 + z1728 = x1728 + z1729 = x1729 + z1730 = x1730 + z1731 = x1731 + z1732 = x1732 + z1733 = x1733 + z1734 = x1734 + z1735 = x1735 + z1736 = x1736 + z1737 = x1737 + z1738 = x1738 + z1739 = x1739 + z1740 = x1740 + z1741 = x1741 + z1742 = x1742 + z1743 = x1743 + z1744 = x1744 + z1745 = x1745 + z1746 = x1746 + z1747 = x1747 + z1748 = x1748 + z1749 = x1749 + z1750 = x1750 + z1751 = x1751 + z1752 = x1752 + z1753 = x1753 + z1754 = x1754 + z1755 = x1755 + z1756 = x1756 + z1757 = x1757 + z1758 = x1758 + z1759 = x1759 + z1760 = x1760 + z1761 = x1761 + z1762 = x1762 + z1763 = x1763 + z1764 = x1764 + z1765 = x1765 + z1766 = x1766 + z1767 = x1767 + z1768 = x1768 + z1769 = x1769 + z1770 = x1770 + z1771 = x1771 + z1772 = x1772 + z1773 = x1773 + z1774 = x1774 + z1775 = x1775 + z1776 = x1776 + z1777 = x1777 + z1778 = x1778 + z1779 = x1779 + z1780 = x1780 + z1781 = x1781 + z1782 = x1782 + z1783 = x1783 + z1784 = x1784 + z1785 = x1785 + z1786 = x1786 + z1787 = x1787 + z1788 = x1788 + z1789 = x1789 + z1790 = x1790 + z1791 = x1791 + z1792 = x1792 + z1793 = x1793 + z1794 = x1794 + z1795 = x1795 + z1796 = x1796 + z1797 = x1797 + z1798 = x1798 + z1799 = x1799 + z1800 = x1800 + z1801 = x1801 + z1802 = x1802 + z1803 = x1803 + z1804 = x1804 + z1805 = x1805 + z1806 = x1806 + z1807 = x1807 + z1808 = x1808 + z1809 = x1809 + z1810 = x1810 + z1811 = x1811 + z1812 = x1812 + z1813 = x1813 + z1814 = x1814 + z1815 = x1815 + z1816 = x1816 + z1817 = x1817 + z1818 = x1818 + z1819 = x1819 + z1820 = x1820 + z1821 = x1821 + z1822 = x1822 + z1823 = x1823 + z1824 = x1824 + z1825 = x1825 + z1826 = x1826 + z1827 = x1827 + z1828 = x1828 + z1829 = x1829 + z1830 = x1830 + z1831 = x1831 + z1832 = x1832 + z1833 = x1833 + z1834 = x1834 + z1835 = x1835 + z1836 = x1836 + z1837 = x1837 + z1838 = x1838 + z1839 = x1839 + z1840 = x1840 + z1841 = x1841 + z1842 = x1842 + z1843 = x1843 + z1844 = x1844 + z1845 = x1845 + z1846 = x1846 + z1847 = x1847 + z1848 = x1848 + z1849 = x1849 + z1850 = x1850 + z1851 = x1851 + z1852 = x1852 + z1853 = x1853 + z1854 = x1854 + z1855 = x1855 + z1856 = x1856 + z1857 = x1857 + z1858 = x1858 + z1859 = x1859 + z1860 = x1860 + z1861 = x1861 + z1862 = x1862 + z1863 = x1863 + z1864 = x1864 + z1865 = x1865 + z1866 = x1866 + z1867 = x1867 + z1868 = x1868 + z1869 = x1869 + z1870 = x1870 + z1871 = x1871 + z1872 = x1872 + z1873 = x1873 + z1874 = x1874 + z1875 = x1875 + z1876 = x1876 + z1877 = x1877 + z1878 = x1878 + z1879 = x1879 + z1880 = x1880 + z1881 = x1881 + z1882 = x1882 + z1883 = x1883 + z1884 = x1884 + z1885 = x1885 + z1886 = x1886 + z1887 = x1887 + z1888 = x1888 + z1889 = x1889 + z1890 = x1890 + z1891 = x1891 + z1892 = x1892 + z1893 = x1893 + z1894 = x1894 + z1895 = x1895 + z1896 = x1896 + z1897 = x1897 + z1898 = x1898 + z1899 = x1899 + z1900 = x1900 + z1901 = x1901 + z1902 = x1902 + z1903 = x1903 + z1904 = x1904 + z1905 = x1905 + z1906 = x1906 + z1907 = x1907 + z1908 = x1908 + z1909 = x1909 + z1910 = x1910 + z1911 = x1911 + z1912 = x1912 + z1913 = x1913 + z1914 = x1914 + z1915 = x1915 + z1916 = x1916 + z1917 = x1917 + z1918 = x1918 + z1919 = x1919 + z1920 = x1920 + z1921 = x1921 + z1922 = x1922 + z1923 = x1923 + z1924 = x1924 + z1925 = x1925 + z1926 = x1926 + z1927 = x1927 + z1928 = x1928 + z1929 = x1929 + z1930 = x1930 + z1931 = x1931 + z1932 = x1932 + z1933 = x1933 + z1934 = x1934 + z1935 = x1935 + z1936 = x1936 + z1937 = x1937 + z1938 = x1938 + z1939 = x1939 + z1940 = x1940 + z1941 = x1941 + z1942 = x1942 + z1943 = x1943 + z1944 = x1944 + z1945 = x1945 + z1946 = x1946 + z1947 = x1947 + z1948 = x1948 + z1949 = x1949 + z1950 = x1950 + z1951 = x1951 + z1952 = x1952 + z1953 = x1953 + z1954 = x1954 + z1955 = x1955 + z1956 = x1956 + z1957 = x1957 + z1958 = x1958 + z1959 = x1959 + z1960 = x1960 + z1961 = x1961 + z1962 = x1962 + z1963 = x1963 + z1964 = x1964 + z1965 = x1965 + z1966 = x1966 + z1967 = x1967 + z1968 = x1968 + z1969 = x1969 + z1970 = x1970 + z1971 = x1971 + z1972 = x1972 + z1973 = x1973 + z1974 = x1974 + z1975 = x1975 + z1976 = x1976 + z1977 = x1977 + z1978 = x1978 + z1979 = x1979 + z1980 = x1980 + z1981 = x1981 + z1982 = x1982 + z1983 = x1983 + z1984 = x1984 + z1985 = x1985 + z1986 = x1986 + z1987 = x1987 + z1988 = x1988 + z1989 = x1989 + z1990 = x1990 + z1991 = x1991 + z1992 = x1992 + z1993 = x1993 + z1994 = x1994 + z1995 = x1995 + z1996 = x1996 + z1997 = x1997 + z1998 = x1998 + z1999 = x1999 + z2000 = x2000 + z2001 = x2001 + z2002 = x2002 + z2003 = x2003 + z2004 = x2004 + z2005 = x2005 + z2006 = x2006 + z2007 = x2007 + z2008 = x2008 + z2009 = x2009 + z2010 = x2010 + z2011 = x2011 + z2012 = x2012 + z2013 = x2013 + z2014 = x2014 + z2015 = x2015 + z2016 = x2016 + z2017 = x2017 + z2018 = x2018 + z2019 = x2019 + z2020 = x2020 + z2021 = x2021 + z2022 = x2022 + z2023 = x2023 + z2024 = x2024 + z2025 = x2025 + z2026 = x2026 + z2027 = x2027 + z2028 = x2028 + z2029 = x2029 + z2030 = x2030 + z2031 = x2031 + z2032 = x2032 + z2033 = x2033 + z2034 = x2034 + z2035 = x2035 + z2036 = x2036 + z2037 = x2037 + z2038 = x2038 + z2039 = x2039 + z2040 = x2040 + z2041 = x2041 + z2042 = x2042 + z2043 = x2043 + z2044 = x2044 + z2045 = x2045 + z2046 = x2046 + z2047 = x2047 + z2048 = x2048 + z2049 = x2049 + z2050 = x2050 + z2051 = x2051 + z2052 = x2052 + z2053 = x2053 + z2054 = x2054 + z2055 = x2055 + z2056 = x2056 + z2057 = x2057 + z2058 = x2058 + z2059 = x2059 + z2060 = x2060 + z2061 = x2061 + z2062 = x2062 + z2063 = x2063 + z2064 = x2064 + z2065 = x2065 + z2066 = x2066 + z2067 = x2067 + z2068 = x2068 + z2069 = x2069 + z2070 = x2070 + z2071 = x2071 + z2072 = x2072 + z2073 = x2073 + z2074 = x2074 + z2075 = x2075 + z2076 = x2076 + z2077 = x2077 + z2078 = x2078 + z2079 = x2079 + z2080 = x2080 + z2081 = x2081 + z2082 = x2082 + z2083 = x2083 + z2084 = x2084 + z2085 = x2085 + z2086 = x2086 + z2087 = x2087 + z2088 = x2088 + z2089 = x2089 + z2090 = x2090 + z2091 = x2091 + z2092 = x2092 + z2093 = x2093 + z2094 = x2094 + z2095 = x2095 + z2096 = x2096 + z2097 = x2097 + z2098 = x2098 + z2099 = x2099 + z2100 = x2100 + z2101 = x2101 + z2102 = x2102 + z2103 = x2103 + z2104 = x2104 + z2105 = x2105 + z2106 = x2106 + z2107 = x2107 + z2108 = x2108 + z2109 = x2109 + z2110 = x2110 + z2111 = x2111 + z2112 = x2112 + z2113 = x2113 + z2114 = x2114 + z2115 = x2115 + z2116 = x2116 + z2117 = x2117 + z2118 = x2118 + z2119 = x2119 + z2120 = x2120 + z2121 = x2121 + z2122 = x2122 + z2123 = x2123 + z2124 = x2124 + z2125 = x2125 + z2126 = x2126 + z2127 = x2127 + z2128 = x2128 + z2129 = x2129 + z2130 = x2130 + z2131 = x2131 + z2132 = x2132 + z2133 = x2133 + z2134 = x2134 + z2135 = x2135 + z2136 = x2136 + z2137 = x2137 + z2138 = x2138 + z2139 = x2139 + z2140 = x2140 + z2141 = x2141 + z2142 = x2142 + z2143 = x2143 + z2144 = x2144 + z2145 = x2145 + z2146 = x2146 + z2147 = x2147 + z2148 = x2148 + z2149 = x2149 + z2150 = x2150 + z2151 = x2151 + z2152 = x2152 + z2153 = x2153 + z2154 = x2154 + z2155 = x2155 + z2156 = x2156 + z2157 = x2157 + z2158 = x2158 + z2159 = x2159 + z2160 = x2160 + z2161 = x2161 + z2162 = x2162 + z2163 = x2163 + z2164 = x2164 + z2165 = x2165 + z2166 = x2166 + z2167 = x2167 + z2168 = x2168 + z2169 = x2169 + z2170 = x2170 + z2171 = x2171 + z2172 = x2172 + z2173 = x2173 + z2174 = x2174 + z2175 = x2175 + z2176 = x2176 + z2177 = x2177 + z2178 = x2178 + z2179 = x2179 + z2180 = x2180 + z2181 = x2181 + z2182 = x2182 + z2183 = x2183 + z2184 = x2184 + z2185 = x2185 + z2186 = x2186 + z2187 = x2187 + z2188 = x2188 + z2189 = x2189 + z2190 = x2190 + z2191 = x2191 + z2192 = x2192 + z2193 = x2193 + z2194 = x2194 + z2195 = x2195 + z2196 = x2196 + z2197 = x2197 + z2198 = x2198 + z2199 = x2199 + z2200 = x2200 + z2201 = x2201 + z2202 = x2202 + z2203 = x2203 + z2204 = x2204 + z2205 = x2205 + z2206 = x2206 + z2207 = x2207 + z2208 = x2208 + z2209 = x2209 + z2210 = x2210 + z2211 = x2211 + z2212 = x2212 + z2213 = x2213 + z2214 = x2214 + z2215 = x2215 + z2216 = x2216 + z2217 = x2217 + z2218 = x2218 + z2219 = x2219 + z2220 = x2220 + z2221 = x2221 + z2222 = x2222 + z2223 = x2223 + z2224 = x2224 + z2225 = x2225 + z2226 = x2226 + z2227 = x2227 + z2228 = x2228 + z2229 = x2229 + z2230 = x2230 + z2231 = x2231 + z2232 = x2232 + z2233 = x2233 + z2234 = x2234 + z2235 = x2235 + z2236 = x2236 + z2237 = x2237 + z2238 = x2238 + z2239 = x2239 + z2240 = x2240 + z2241 = x2241 + z2242 = x2242 + z2243 = x2243 + z2244 = x2244 + z2245 = x2245 + z2246 = x2246 + z2247 = x2247 + z2248 = x2248 + z2249 = x2249 + z2250 = x2250 + z2251 = x2251 + z2252 = x2252 + z2253 = x2253 + z2254 = x2254 + z2255 = x2255 + z2256 = x2256 + z2257 = x2257 + z2258 = x2258 + z2259 = x2259 + z2260 = x2260 + z2261 = x2261 + z2262 = x2262 + z2263 = x2263 + z2264 = x2264 + z2265 = x2265 + z2266 = x2266 + z2267 = x2267 + z2268 = x2268 + z2269 = x2269 + z2270 = x2270 + z2271 = x2271 + z2272 = x2272 + z2273 = x2273 + z2274 = x2274 + z2275 = x2275 + z2276 = x2276 + z2277 = x2277 + z2278 = x2278 + z2279 = x2279 + z2280 = x2280 + z2281 = x2281 + z2282 = x2282 + z2283 = x2283 + z2284 = x2284 + z2285 = x2285 + z2286 = x2286 + z2287 = x2287 + z2288 = x2288 + z2289 = x2289 + z2290 = x2290 + z2291 = x2291 + z2292 = x2292 + z2293 = x2293 + z2294 = x2294 + z2295 = x2295 + z2296 = x2296 + z2297 = x2297 + z2298 = x2298 + z2299 = x2299 + z2300 = x2300 + z2301 = x2301 + z2302 = x2302 + z2303 = x2303 + z2304 = x2304 + z2305 = x2305 + z2306 = x2306 + z2307 = x2307 + z2308 = x2308 + z2309 = x2309 + z2310 = x2310 + z2311 = x2311 + z2312 = x2312 + z2313 = x2313 + z2314 = x2314 + z2315 = x2315 + z2316 = x2316 + z2317 = x2317 + z2318 = x2318 + z2319 = x2319 + z2320 = x2320 + z2321 = x2321 + z2322 = x2322 + z2323 = x2323 + z2324 = x2324 + z2325 = x2325 + z2326 = x2326 + z2327 = x2327 + z2328 = x2328 + z2329 = x2329 + z2330 = x2330 + z2331 = x2331 + z2332 = x2332 + z2333 = x2333 + z2334 = x2334 + z2335 = x2335 + z2336 = x2336 + z2337 = x2337 + z2338 = x2338 + z2339 = x2339 + z2340 = x2340 + z2341 = x2341 + z2342 = x2342 + z2343 = x2343 + z2344 = x2344 + z2345 = x2345 + z2346 = x2346 + z2347 = x2347 + z2348 = x2348 + z2349 = x2349 + z2350 = x2350 + z2351 = x2351 + z2352 = x2352 + z2353 = x2353 + z2354 = x2354 + z2355 = x2355 + z2356 = x2356 + z2357 = x2357 + z2358 = x2358 + z2359 = x2359 + z2360 = x2360 + z2361 = x2361 + z2362 = x2362 + z2363 = x2363 + z2364 = x2364 + z2365 = x2365 + z2366 = x2366 + z2367 = x2367 + z2368 = x2368 + z2369 = x2369 + z2370 = x2370 + z2371 = x2371 + z2372 = x2372 + z2373 = x2373 + z2374 = x2374 + z2375 = x2375 + z2376 = x2376 + z2377 = x2377 + z2378 = x2378 + z2379 = x2379 + z2380 = x2380 + z2381 = x2381 + z2382 = x2382 + z2383 = x2383 + z2384 = x2384 + z2385 = x2385 + z2386 = x2386 + z2387 = x2387 + z2388 = x2388 + z2389 = x2389 + z2390 = x2390 + z2391 = x2391 + z2392 = x2392 + z2393 = x2393 + z2394 = x2394 + z2395 = x2395 + z2396 = x2396 + z2397 = x2397 + z2398 = x2398 + z2399 = x2399 + z2400 = x2400 + z2401 = x2401 + z2402 = x2402 + z2403 = x2403 + z2404 = x2404 + z2405 = x2405 + z2406 = x2406 + z2407 = x2407 + z2408 = x2408 + z2409 = x2409 + z2410 = x2410 + z2411 = x2411 + z2412 = x2412 + z2413 = x2413 + z2414 = x2414 + z2415 = x2415 + z2416 = x2416 + z2417 = x2417 + z2418 = x2418 + z2419 = x2419 + z2420 = x2420 + z2421 = x2421 + z2422 = x2422 + z2423 = x2423 + z2424 = x2424 + z2425 = x2425 + z2426 = x2426 + z2427 = x2427 + z2428 = x2428 + z2429 = x2429 + z2430 = x2430 + z2431 = x2431 + z2432 = x2432 + z2433 = x2433 + z2434 = x2434 + z2435 = x2435 + z2436 = x2436 + z2437 = x2437 + z2438 = x2438 + z2439 = x2439 + z2440 = x2440 + z2441 = x2441 + z2442 = x2442 + z2443 = x2443 + z2444 = x2444 + z2445 = x2445 + z2446 = x2446 + z2447 = x2447 + z2448 = x2448 + z2449 = x2449 + z2450 = x2450 + z2451 = x2451 + z2452 = x2452 + z2453 = x2453 + z2454 = x2454 + z2455 = x2455 + z2456 = x2456 + z2457 = x2457 + z2458 = x2458 + z2459 = x2459 + z2460 = x2460 + z2461 = x2461 + z2462 = x2462 + z2463 = x2463 + z2464 = x2464 + z2465 = x2465 + z2466 = x2466 + z2467 = x2467 + z2468 = x2468 + z2469 = x2469 + z2470 = x2470 + z2471 = x2471 + z2472 = x2472 + z2473 = x2473 + z2474 = x2474 + z2475 = x2475 + z2476 = x2476 + z2477 = x2477 + z2478 = x2478 + z2479 = x2479 + z2480 = x2480 + z2481 = x2481 + z2482 = x2482 + z2483 = x2483 + z2484 = x2484 + z2485 = x2485 + z2486 = x2486 + z2487 = x2487 + z2488 = x2488 + z2489 = x2489 + z2490 = x2490 + z2491 = x2491 + z2492 = x2492 + z2493 = x2493 + z2494 = x2494 + z2495 = x2495 + z2496 = x2496 + z2497 = x2497 + z2498 = x2498 + z2499 = x2499 + z2500 = x2500 + z2501 = x2501 + z2502 = x2502 + z2503 = x2503 + z2504 = x2504 + z2505 = x2505 + z2506 = x2506 + z2507 = x2507 + z2508 = x2508 + z2509 = x2509 + z2510 = x2510 + z2511 = x2511 + z2512 = x2512 + z2513 = x2513 + z2514 = x2514 + z2515 = x2515 + z2516 = x2516 + z2517 = x2517 + z2518 = x2518 + z2519 = x2519 + z2520 = x2520 + z2521 = x2521 + z2522 = x2522 + z2523 = x2523 + z2524 = x2524 + z2525 = x2525 + z2526 = x2526 + z2527 = x2527 + z2528 = x2528 + z2529 = x2529 + z2530 = x2530 + z2531 = x2531 + z2532 = x2532 + z2533 = x2533 + z2534 = x2534 + z2535 = x2535 + z2536 = x2536 + z2537 = x2537 + z2538 = x2538 + z2539 = x2539 + z2540 = x2540 + z2541 = x2541 + z2542 = x2542 + z2543 = x2543 + z2544 = x2544 + z2545 = x2545 + z2546 = x2546 + z2547 = x2547 + z2548 = x2548 + z2549 = x2549 + z2550 = x2550 + z2551 = x2551 + z2552 = x2552 + z2553 = x2553 + z2554 = x2554 + z2555 = x2555 + z2556 = x2556 + z2557 = x2557 + z2558 = x2558 + z2559 = x2559 + z2560 = x2560 + z2561 = x2561 + z2562 = x2562 + z2563 = x2563 + z2564 = x2564 + z2565 = x2565 + z2566 = x2566 + z2567 = x2567 + z2568 = x2568 + z2569 = x2569 + z2570 = x2570 + z2571 = x2571 + z2572 = x2572 + z2573 = x2573 + z2574 = x2574 + z2575 = x2575 + z2576 = x2576 + z2577 = x2577 + z2578 = x2578 + z2579 = x2579 + z2580 = x2580 + z2581 = x2581 + z2582 = x2582 + z2583 = x2583 + z2584 = x2584 + z2585 = x2585 + z2586 = x2586 + z2587 = x2587 + z2588 = x2588 + z2589 = x2589 + z2590 = x2590 + z2591 = x2591 + z2592 = x2592 + z2593 = x2593 + z2594 = x2594 + z2595 = x2595 + z2596 = x2596 + z2597 = x2597 + z2598 = x2598 + z2599 = x2599 + z2600 = x2600 + z2601 = x2601 + z2602 = x2602 + z2603 = x2603 + z2604 = x2604 + z2605 = x2605 + z2606 = x2606 + z2607 = x2607 + z2608 = x2608 + z2609 = x2609 + z2610 = x2610 + z2611 = x2611 + z2612 = x2612 + z2613 = x2613 + z2614 = x2614 + z2615 = x2615 + z2616 = x2616 + z2617 = x2617 + z2618 = x2618 + z2619 = x2619 + z2620 = x2620 + z2621 = x2621 + z2622 = x2622 + z2623 = x2623 + z2624 = x2624 + z2625 = x2625 + z2626 = x2626 + z2627 = x2627 + z2628 = x2628 + z2629 = x2629 + z2630 = x2630 + z2631 = x2631 + z2632 = x2632 + z2633 = x2633 + z2634 = x2634 + z2635 = x2635 + z2636 = x2636 + z2637 = x2637 + z2638 = x2638 + z2639 = x2639 + z2640 = x2640 + z2641 = x2641 + z2642 = x2642 + z2643 = x2643 + z2644 = x2644 + z2645 = x2645 + z2646 = x2646 + z2647 = x2647 + z2648 = x2648 + z2649 = x2649 + z2650 = x2650 + z2651 = x2651 + z2652 = x2652 + z2653 = x2653 + z2654 = x2654 + z2655 = x2655 + z2656 = x2656 + z2657 = x2657 + z2658 = x2658 + z2659 = x2659 + z2660 = x2660 + z2661 = x2661 + z2662 = x2662 + z2663 = x2663 + z2664 = x2664 + z2665 = x2665 + z2666 = x2666 + z2667 = x2667 + z2668 = x2668 + z2669 = x2669 + z2670 = x2670 + z2671 = x2671 + z2672 = x2672 + z2673 = x2673 + z2674 = x2674 + z2675 = x2675 + z2676 = x2676 + z2677 = x2677 + z2678 = x2678 + z2679 = x2679 + z2680 = x2680 + z2681 = x2681 + z2682 = x2682 + z2683 = x2683 + z2684 = x2684 + z2685 = x2685 + z2686 = x2686 + z2687 = x2687 + z2688 = x2688 + z2689 = x2689 + z2690 = x2690 + z2691 = x2691 + z2692 = x2692 + z2693 = x2693 + z2694 = x2694 + z2695 = x2695 + z2696 = x2696 + z2697 = x2697 + z2698 = x2698 + z2699 = x2699 + z2700 = x2700 + z2701 = x2701 + z2702 = x2702 + z2703 = x2703 + z2704 = x2704 + z2705 = x2705 + z2706 = x2706 + z2707 = x2707 + z2708 = x2708 + z2709 = x2709 + z2710 = x2710 + z2711 = x2711 + z2712 = x2712 + z2713 = x2713 + z2714 = x2714 + z2715 = x2715 + z2716 = x2716 + z2717 = x2717 + z2718 = x2718 + z2719 = x2719 + z2720 = x2720 + z2721 = x2721 + z2722 = x2722 + z2723 = x2723 + z2724 = x2724 + z2725 = x2725 + z2726 = x2726 + z2727 = x2727 + z2728 = x2728 + z2729 = x2729 + z2730 = x2730 + z2731 = x2731 + z2732 = x2732 + z2733 = x2733 + z2734 = x2734 + z2735 = x2735 + z2736 = x2736 + z2737 = x2737 + z2738 = x2738 + z2739 = x2739 + z2740 = x2740 + z2741 = x2741 + z2742 = x2742 + z2743 = x2743 + z2744 = x2744 + z2745 = x2745 + z2746 = x2746 + z2747 = x2747 + z2748 = x2748 + z2749 = x2749 + z2750 = x2750 + z2751 = x2751 + z2752 = x2752 + z2753 = x2753 + z2754 = x2754 + z2755 = x2755 + z2756 = x2756 + z2757 = x2757 + z2758 = x2758 + z2759 = x2759 + z2760 = x2760 + z2761 = x2761 + z2762 = x2762 + z2763 = x2763 + z2764 = x2764 + z2765 = x2765 + z2766 = x2766 + z2767 = x2767 + z2768 = x2768 + z2769 = x2769 + z2770 = x2770 + z2771 = x2771 + z2772 = x2772 + z2773 = x2773 + z2774 = x2774 + z2775 = x2775 + z2776 = x2776 + z2777 = x2777 + z2778 = x2778 + z2779 = x2779 + z2780 = x2780 + z2781 = x2781 + z2782 = x2782 + z2783 = x2783 + z2784 = x2784 + z2785 = x2785 + z2786 = x2786 + z2787 = x2787 + z2788 = x2788 + z2789 = x2789 + z2790 = x2790 + z2791 = x2791 + z2792 = x2792 + z2793 = x2793 + z2794 = x2794 + z2795 = x2795 + z2796 = x2796 + z2797 = x2797 + z2798 = x2798 + z2799 = x2799 + z2800 = x2800 + z2801 = x2801 + z2802 = x2802 + z2803 = x2803 + z2804 = x2804 + z2805 = x2805 + z2806 = x2806 + z2807 = x2807 + z2808 = x2808 + z2809 = x2809 + z2810 = x2810 + z2811 = x2811 + z2812 = x2812 + z2813 = x2813 + z2814 = x2814 + z2815 = x2815 + z2816 = x2816 + z2817 = x2817 + z2818 = x2818 + z2819 = x2819 + z2820 = x2820 + z2821 = x2821 + z2822 = x2822 + z2823 = x2823 + z2824 = x2824 + z2825 = x2825 + z2826 = x2826 + z2827 = x2827 + z2828 = x2828 + z2829 = x2829 + z2830 = x2830 + z2831 = x2831 + z2832 = x2832 + z2833 = x2833 + z2834 = x2834 + z2835 = x2835 + z2836 = x2836 + z2837 = x2837 + z2838 = x2838 + z2839 = x2839 + z2840 = x2840 + z2841 = x2841 + z2842 = x2842 + z2843 = x2843 + z2844 = x2844 + z2845 = x2845 + z2846 = x2846 + z2847 = x2847 + z2848 = x2848 + z2849 = x2849 + z2850 = x2850 + z2851 = x2851 + z2852 = x2852 + z2853 = x2853 + z2854 = x2854 + z2855 = x2855 + z2856 = x2856 + z2857 = x2857 + z2858 = x2858 + z2859 = x2859 + z2860 = x2860 + z2861 = x2861 + z2862 = x2862 + z2863 = x2863 + z2864 = x2864 + z2865 = x2865 + z2866 = x2866 + z2867 = x2867 + z2868 = x2868 + z2869 = x2869 + z2870 = x2870 + z2871 = x2871 + z2872 = x2872 + z2873 = x2873 + z2874 = x2874 + z2875 = x2875 + z2876 = x2876 + z2877 = x2877 + z2878 = x2878 + z2879 = x2879 + z2880 = x2880 + z2881 = x2881 + z2882 = x2882 + z2883 = x2883 + z2884 = x2884 + z2885 = x2885 + z2886 = x2886 + z2887 = x2887 + z2888 = x2888 + z2889 = x2889 + z2890 = x2890 + z2891 = x2891 + z2892 = x2892 + z2893 = x2893 + z2894 = x2894 + z2895 = x2895 + z2896 = x2896 + z2897 = x2897 + z2898 = x2898 + z2899 = x2899 + z2900 = x2900 + z2901 = x2901 + z2902 = x2902 + z2903 = x2903 + z2904 = x2904 + z2905 = x2905 + z2906 = x2906 + z2907 = x2907 + z2908 = x2908 + z2909 = x2909 + z2910 = x2910 + z2911 = x2911 + z2912 = x2912 + z2913 = x2913 + z2914 = x2914 + z2915 = x2915 + z2916 = x2916 + z2917 = x2917 + z2918 = x2918 + z2919 = x2919 + z2920 = x2920 + z2921 = x2921 + z2922 = x2922 + z2923 = x2923 + z2924 = x2924 + z2925 = x2925 + z2926 = x2926 + z2927 = x2927 + z2928 = x2928 + z2929 = x2929 + z2930 = x2930 + z2931 = x2931 + z2932 = x2932 + z2933 = x2933 + z2934 = x2934 + z2935 = x2935 + z2936 = x2936 + z2937 = x2937 + z2938 = x2938 + z2939 = x2939 + z2940 = x2940 + z2941 = x2941 + z2942 = x2942 + z2943 = x2943 + z2944 = x2944 + z2945 = x2945 + z2946 = x2946 + z2947 = x2947 + z2948 = x2948 + z2949 = x2949 + z2950 = x2950 + z2951 = x2951 + z2952 = x2952 + z2953 = x2953 + z2954 = x2954 + z2955 = x2955 + z2956 = x2956 + z2957 = x2957 + z2958 = x2958 + z2959 = x2959 + z2960 = x2960 + z2961 = x2961 + z2962 = x2962 + z2963 = x2963 + z2964 = x2964 + z2965 = x2965 + z2966 = x2966 + z2967 = x2967 + z2968 = x2968 + z2969 = x2969 + z2970 = x2970 + z2971 = x2971 + z2972 = x2972 + z2973 = x2973 + z2974 = x2974 + z2975 = x2975 + z2976 = x2976 + z2977 = x2977 + z2978 = x2978 + z2979 = x2979 + z2980 = x2980 + z2981 = x2981 + z2982 = x2982 + z2983 = x2983 + z2984 = x2984 + z2985 = x2985 + z2986 = x2986 + z2987 = x2987 + z2988 = x2988 + z2989 = x2989 + z2990 = x2990 + z2991 = x2991 + z2992 = x2992 + z2993 = x2993 + z2994 = x2994 + z2995 = x2995 + z2996 = x2996 + z2997 = x2997 + z2998 = x2998 + z2999 = x2999 + z3000 = x3000 + z3001 = x3001 + z3002 = x3002 + z3003 = x3003 + z3004 = x3004 + z3005 = x3005 + z3006 = x3006 + z3007 = x3007 + z3008 = x3008 + z3009 = x3009 + z3010 = x3010 + z3011 = x3011 + z3012 = x3012 + z3013 = x3013 + z3014 = x3014 + z3015 = x3015 + z3016 = x3016 + z3017 = x3017 + z3018 = x3018 + z3019 = x3019 + z3020 = x3020 + z3021 = x3021 + z3022 = x3022 + z3023 = x3023 + z3024 = x3024 + z3025 = x3025 + z3026 = x3026 + z3027 = x3027 + z3028 = x3028 + z3029 = x3029 + z3030 = x3030 + z3031 = x3031 + z3032 = x3032 + z3033 = x3033 + z3034 = x3034 + z3035 = x3035 + z3036 = x3036 + z3037 = x3037 + z3038 = x3038 + z3039 = x3039 + z3040 = x3040 + z3041 = x3041 + z3042 = x3042 + z3043 = x3043 + z3044 = x3044 + z3045 = x3045 + z3046 = x3046 + z3047 = x3047 + z3048 = x3048 + z3049 = x3049 + z3050 = x3050 + z3051 = x3051 + z3052 = x3052 + z3053 = x3053 + z3054 = x3054 + z3055 = x3055 + z3056 = x3056 + z3057 = x3057 + z3058 = x3058 + z3059 = x3059 + z3060 = x3060 + z3061 = x3061 + z3062 = x3062 + z3063 = x3063 + z3064 = x3064 + z3065 = x3065 + z3066 = x3066 + z3067 = x3067 + z3068 = x3068 + z3069 = x3069 + z3070 = x3070 + z3071 = x3071 + z3072 = x3072 + z3073 = x3073 + z3074 = x3074 + z3075 = x3075 + z3076 = x3076 + z3077 = x3077 + z3078 = x3078 + z3079 = x3079 + z3080 = x3080 + z3081 = x3081 + z3082 = x3082 + z3083 = x3083 + z3084 = x3084 + z3085 = x3085 + z3086 = x3086 + z3087 = x3087 + z3088 = x3088 + z3089 = x3089 + z3090 = x3090 + z3091 = x3091 + z3092 = x3092 + z3093 = x3093 + z3094 = x3094 + z3095 = x3095 + z3096 = x3096 + z3097 = x3097 + z3098 = x3098 + z3099 = x3099 + z3100 = x3100 + z3101 = x3101 + z3102 = x3102 + z3103 = x3103 + z3104 = x3104 + z3105 = x3105 + z3106 = x3106 + z3107 = x3107 + z3108 = x3108 + z3109 = x3109 + z3110 = x3110 + z3111 = x3111 + z3112 = x3112 + z3113 = x3113 + z3114 = x3114 + z3115 = x3115 + z3116 = x3116 + z3117 = x3117 + z3118 = x3118 + z3119 = x3119 + z3120 = x3120 + z3121 = x3121 + z3122 = x3122 + z3123 = x3123 + z3124 = x3124 + z3125 = x3125 + z3126 = x3126 + z3127 = x3127 + z3128 = x3128 + z3129 = x3129 + z3130 = x3130 + z3131 = x3131 + z3132 = x3132 + z3133 = x3133 + z3134 = x3134 + z3135 = x3135 + z3136 = x3136 + z3137 = x3137 + z3138 = x3138 + z3139 = x3139 + z3140 = x3140 + z3141 = x3141 + z3142 = x3142 + z3143 = x3143 + z3144 = x3144 + z3145 = x3145 + z3146 = x3146 + z3147 = x3147 + z3148 = x3148 + z3149 = x3149 + z3150 = x3150 + z3151 = x3151 + z3152 = x3152 + z3153 = x3153 + z3154 = x3154 + z3155 = x3155 + z3156 = x3156 + z3157 = x3157 + z3158 = x3158 + z3159 = x3159 + z3160 = x3160 + z3161 = x3161 + z3162 = x3162 + z3163 = x3163 + z3164 = x3164 + z3165 = x3165 + z3166 = x3166 + z3167 = x3167 + z3168 = x3168 + z3169 = x3169 + z3170 = x3170 + z3171 = x3171 + z3172 = x3172 + z3173 = x3173 + z3174 = x3174 + z3175 = x3175 + z3176 = x3176 + z3177 = x3177 + z3178 = x3178 + z3179 = x3179 + z3180 = x3180 + z3181 = x3181 + z3182 = x3182 + z3183 = x3183 + z3184 = x3184 + z3185 = x3185 + z3186 = x3186 + z3187 = x3187 + z3188 = x3188 + z3189 = x3189 + z3190 = x3190 + z3191 = x3191 + z3192 = x3192 + z3193 = x3193 + z3194 = x3194 + z3195 = x3195 + z3196 = x3196 + z3197 = x3197 + z3198 = x3198 + z3199 = x3199 + z3200 = x3200 + z3201 = x3201 + z3202 = x3202 + z3203 = x3203 + z3204 = x3204 + z3205 = x3205 + z3206 = x3206 + z3207 = x3207 + z3208 = x3208 + z3209 = x3209 + z3210 = x3210 + z3211 = x3211 + z3212 = x3212 + z3213 = x3213 + z3214 = x3214 + z3215 = x3215 + z3216 = x3216 + z3217 = x3217 + z3218 = x3218 + z3219 = x3219 + z3220 = x3220 + z3221 = x3221 + z3222 = x3222 + z3223 = x3223 + z3224 = x3224 + z3225 = x3225 + z3226 = x3226 + z3227 = x3227 + z3228 = x3228 + z3229 = x3229 + z3230 = x3230 + z3231 = x3231 + z3232 = x3232 + z3233 = x3233 + z3234 = x3234 + z3235 = x3235 + z3236 = x3236 + z3237 = x3237 + z3238 = x3238 + z3239 = x3239 + z3240 = x3240 + z3241 = x3241 + z3242 = x3242 + z3243 = x3243 + z3244 = x3244 + z3245 = x3245 + z3246 = x3246 + z3247 = x3247 + z3248 = x3248 + z3249 = x3249 + z3250 = x3250 + z3251 = x3251 + z3252 = x3252 + z3253 = x3253 + z3254 = x3254 + z3255 = x3255 + z3256 = x3256 + z3257 = x3257 + z3258 = x3258 + z3259 = x3259 + z3260 = x3260 + z3261 = x3261 + z3262 = x3262 + z3263 = x3263 + z3264 = x3264 + z3265 = x3265 + z3266 = x3266 + z3267 = x3267 + z3268 = x3268 + z3269 = x3269 + z3270 = x3270 + z3271 = x3271 + z3272 = x3272 + z3273 = x3273 + z3274 = x3274 + z3275 = x3275 + z3276 = x3276 + z3277 = x3277 + z3278 = x3278 + z3279 = x3279 + z3280 = x3280 + z3281 = x3281 + z3282 = x3282 + z3283 = x3283 + z3284 = x3284 + z3285 = x3285 + z3286 = x3286 + z3287 = x3287 + z3288 = x3288 + z3289 = x3289 + z3290 = x3290 + z3291 = x3291 + z3292 = x3292 + z3293 = x3293 + z3294 = x3294 + z3295 = x3295 + z3296 = x3296 + z3297 = x3297 + z3298 = x3298 + z3299 = x3299 + z3300 = x3300 + z3301 = x3301 + z3302 = x3302 + z3303 = x3303 + z3304 = x3304 + z3305 = x3305 + z3306 = x3306 + z3307 = x3307 + z3308 = x3308 + z3309 = x3309 + z3310 = x3310 + z3311 = x3311 + z3312 = x3312 + z3313 = x3313 + z3314 = x3314 + z3315 = x3315 + z3316 = x3316 + z3317 = x3317 + z3318 = x3318 + z3319 = x3319 + z3320 = x3320 + z3321 = x3321 + z3322 = x3322 + z3323 = x3323 + z3324 = x3324 + z3325 = x3325 + z3326 = x3326 + z3327 = x3327 + z3328 = x3328 + z3329 = x3329 + z3330 = x3330 + z3331 = x3331 + z3332 = x3332 + z3333 = x3333 + z3334 = x3334 + z3335 = x3335 + z3336 = x3336 + z3337 = x3337 + z3338 = x3338 + z3339 = x3339 + z3340 = x3340 + z3341 = x3341 + z3342 = x3342 + z3343 = x3343 + z3344 = x3344 + z3345 = x3345 + z3346 = x3346 + z3347 = x3347 + z3348 = x3348 + z3349 = x3349 + z3350 = x3350 + z3351 = x3351 + z3352 = x3352 + z3353 = x3353 + z3354 = x3354 + z3355 = x3355 + z3356 = x3356 + z3357 = x3357 + z3358 = x3358 + z3359 = x3359 + z3360 = x3360 + z3361 = x3361 + z3362 = x3362 + z3363 = x3363 + z3364 = x3364 + z3365 = x3365 + z3366 = x3366 + z3367 = x3367 + z3368 = x3368 + z3369 = x3369 + z3370 = x3370 + z3371 = x3371 + z3372 = x3372 + z3373 = x3373 + z3374 = x3374 + z3375 = x3375 + z3376 = x3376 + z3377 = x3377 + z3378 = x3378 + z3379 = x3379 + z3380 = x3380 + z3381 = x3381 + z3382 = x3382 + z3383 = x3383 + z3384 = x3384 + z3385 = x3385 + z3386 = x3386 + z3387 = x3387 + z3388 = x3388 + z3389 = x3389 + z3390 = x3390 + z3391 = x3391 + z3392 = x3392 + z3393 = x3393 + z3394 = x3394 + z3395 = x3395 + z3396 = x3396 + z3397 = x3397 + z3398 = x3398 + z3399 = x3399 + z3400 = x3400 + z3401 = x3401 + z3402 = x3402 + z3403 = x3403 + z3404 = x3404 + z3405 = x3405 + z3406 = x3406 + z3407 = x3407 + z3408 = x3408 + z3409 = x3409 + z3410 = x3410 + z3411 = x3411 + z3412 = x3412 + z3413 = x3413 + z3414 = x3414 + z3415 = x3415 + z3416 = x3416 + z3417 = x3417 + z3418 = x3418 + z3419 = x3419 + z3420 = x3420 + z3421 = x3421 + z3422 = x3422 + z3423 = x3423 + z3424 = x3424 + z3425 = x3425 + z3426 = x3426 + z3427 = x3427 + z3428 = x3428 + z3429 = x3429 + z3430 = x3430 + z3431 = x3431 + z3432 = x3432 + z3433 = x3433 + z3434 = x3434 + z3435 = x3435 + z3436 = x3436 + z3437 = x3437 + z3438 = x3438 + z3439 = x3439 + z3440 = x3440 + z3441 = x3441 + z3442 = x3442 + z3443 = x3443 + z3444 = x3444 + z3445 = x3445 + z3446 = x3446 + z3447 = x3447 + z3448 = x3448 + z3449 = x3449 + z3450 = x3450 + z3451 = x3451 + z3452 = x3452 + z3453 = x3453 + z3454 = x3454 + z3455 = x3455 + z3456 = x3456 + z3457 = x3457 + z3458 = x3458 + z3459 = x3459 + z3460 = x3460 + z3461 = x3461 + z3462 = x3462 + z3463 = x3463 + z3464 = x3464 + z3465 = x3465 + z3466 = x3466 + z3467 = x3467 + z3468 = x3468 + z3469 = x3469 + z3470 = x3470 + z3471 = x3471 + z3472 = x3472 + z3473 = x3473 + z3474 = x3474 + z3475 = x3475 + z3476 = x3476 + z3477 = x3477 + z3478 = x3478 + z3479 = x3479 + z3480 = x3480 + z3481 = x3481 + z3482 = x3482 + z3483 = x3483 + z3484 = x3484 + z3485 = x3485 + z3486 = x3486 + z3487 = x3487 + z3488 = x3488 + z3489 = x3489 + z3490 = x3490 + z3491 = x3491 + z3492 = x3492 + z3493 = x3493 + z3494 = x3494 + z3495 = x3495 + z3496 = x3496 + z3497 = x3497 + z3498 = x3498 + z3499 = x3499 + z3500 = x3500 + z3501 = x3501 + z3502 = x3502 + z3503 = x3503 + z3504 = x3504 + z3505 = x3505 + z3506 = x3506 + z3507 = x3507 + z3508 = x3508 + z3509 = x3509 + z3510 = x3510 + z3511 = x3511 + z3512 = x3512 + z3513 = x3513 + z3514 = x3514 + z3515 = x3515 + z3516 = x3516 + z3517 = x3517 + z3518 = x3518 + z3519 = x3519 + z3520 = x3520 + z3521 = x3521 + z3522 = x3522 + z3523 = x3523 + z3524 = x3524 + z3525 = x3525 + z3526 = x3526 + z3527 = x3527 + z3528 = x3528 + z3529 = x3529 + z3530 = x3530 + z3531 = x3531 + z3532 = x3532 + z3533 = x3533 + z3534 = x3534 + z3535 = x3535 + z3536 = x3536 + z3537 = x3537 + z3538 = x3538 + z3539 = x3539 + z3540 = x3540 + z3541 = x3541 + z3542 = x3542 + z3543 = x3543 + z3544 = x3544 + z3545 = x3545 + z3546 = x3546 + z3547 = x3547 + z3548 = x3548 + z3549 = x3549 + z3550 = x3550 + z3551 = x3551 + z3552 = x3552 + z3553 = x3553 + z3554 = x3554 + z3555 = x3555 + z3556 = x3556 + z3557 = x3557 + z3558 = x3558 + z3559 = x3559 + z3560 = x3560 + z3561 = x3561 + z3562 = x3562 + z3563 = x3563 + z3564 = x3564 + z3565 = x3565 + z3566 = x3566 + z3567 = x3567 + z3568 = x3568 + z3569 = x3569 + z3570 = x3570 + z3571 = x3571 + z3572 = x3572 + z3573 = x3573 + z3574 = x3574 + z3575 = x3575 + z3576 = x3576 + z3577 = x3577 + z3578 = x3578 + z3579 = x3579 + z3580 = x3580 + z3581 = x3581 + z3582 = x3582 + z3583 = x3583 + z3584 = x3584 + z3585 = x3585 + z3586 = x3586 + z3587 = x3587 + z3588 = x3588 + z3589 = x3589 + z3590 = x3590 + z3591 = x3591 + z3592 = x3592 + z3593 = x3593 + z3594 = x3594 + z3595 = x3595 + z3596 = x3596 + z3597 = x3597 + z3598 = x3598 + z3599 = x3599 + z3600 = x3600 + z3601 = x3601 + z3602 = x3602 + z3603 = x3603 + z3604 = x3604 + z3605 = x3605 + z3606 = x3606 + z3607 = x3607 + z3608 = x3608 + z3609 = x3609 + z3610 = x3610 + z3611 = x3611 + z3612 = x3612 + z3613 = x3613 + z3614 = x3614 + z3615 = x3615 + z3616 = x3616 + z3617 = x3617 + z3618 = x3618 + z3619 = x3619 + z3620 = x3620 + z3621 = x3621 + z3622 = x3622 + z3623 = x3623 + z3624 = x3624 + z3625 = x3625 + z3626 = x3626 + z3627 = x3627 + z3628 = x3628 + z3629 = x3629 + z3630 = x3630 + z3631 = x3631 + z3632 = x3632 + z3633 = x3633 + z3634 = x3634 + z3635 = x3635 + z3636 = x3636 + z3637 = x3637 + z3638 = x3638 + z3639 = x3639 + z3640 = x3640 + z3641 = x3641 + z3642 = x3642 + z3643 = x3643 + z3644 = x3644 + z3645 = x3645 + z3646 = x3646 + z3647 = x3647 + z3648 = x3648 + z3649 = x3649 + z3650 = x3650 + z3651 = x3651 + z3652 = x3652 + z3653 = x3653 + z3654 = x3654 + z3655 = x3655 + z3656 = x3656 + z3657 = x3657 + z3658 = x3658 + z3659 = x3659 + z3660 = x3660 + z3661 = x3661 + z3662 = x3662 + z3663 = x3663 + z3664 = x3664 + z3665 = x3665 + z3666 = x3666 + z3667 = x3667 + z3668 = x3668 + z3669 = x3669 + z3670 = x3670 + z3671 = x3671 + z3672 = x3672 + z3673 = x3673 + z3674 = x3674 + z3675 = x3675 + z3676 = x3676 + z3677 = x3677 + z3678 = x3678 + z3679 = x3679 + z3680 = x3680 + z3681 = x3681 + z3682 = x3682 + z3683 = x3683 + z3684 = x3684 + z3685 = x3685 + z3686 = x3686 + z3687 = x3687 + z3688 = x3688 + z3689 = x3689 + z3690 = x3690 + z3691 = x3691 + z3692 = x3692 + z3693 = x3693 + z3694 = x3694 + z3695 = x3695 + z3696 = x3696 + z3697 = x3697 + z3698 = x3698 + z3699 = x3699 + z3700 = x3700 + z3701 = x3701 + z3702 = x3702 + z3703 = x3703 + z3704 = x3704 + z3705 = x3705 + z3706 = x3706 + z3707 = x3707 + z3708 = x3708 + z3709 = x3709 + z3710 = x3710 + z3711 = x3711 + z3712 = x3712 + z3713 = x3713 + z3714 = x3714 + z3715 = x3715 + z3716 = x3716 + z3717 = x3717 + z3718 = x3718 + z3719 = x3719 + z3720 = x3720 + z3721 = x3721 + z3722 = x3722 + z3723 = x3723 + z3724 = x3724 + z3725 = x3725 + z3726 = x3726 + z3727 = x3727 + z3728 = x3728 + z3729 = x3729 + z3730 = x3730 + z3731 = x3731 + z3732 = x3732 + z3733 = x3733 + z3734 = x3734 + z3735 = x3735 + z3736 = x3736 + z3737 = x3737 + z3738 = x3738 + z3739 = x3739 + z3740 = x3740 + z3741 = x3741 + z3742 = x3742 + z3743 = x3743 + z3744 = x3744 + z3745 = x3745 + z3746 = x3746 + z3747 = x3747 + z3748 = x3748 + z3749 = x3749 + z3750 = x3750 + z3751 = x3751 + z3752 = x3752 + z3753 = x3753 + z3754 = x3754 + z3755 = x3755 + z3756 = x3756 + z3757 = x3757 + z3758 = x3758 + z3759 = x3759 + z3760 = x3760 + z3761 = x3761 + z3762 = x3762 + z3763 = x3763 + z3764 = x3764 + z3765 = x3765 + z3766 = x3766 + z3767 = x3767 + z3768 = x3768 + z3769 = x3769 + z3770 = x3770 + z3771 = x3771 + z3772 = x3772 + z3773 = x3773 + z3774 = x3774 + z3775 = x3775 + z3776 = x3776 + z3777 = x3777 + z3778 = x3778 + z3779 = x3779 + z3780 = x3780 + z3781 = x3781 + z3782 = x3782 + z3783 = x3783 + z3784 = x3784 + z3785 = x3785 + z3786 = x3786 + z3787 = x3787 + z3788 = x3788 + z3789 = x3789 + z3790 = x3790 + z3791 = x3791 + z3792 = x3792 + z3793 = x3793 + z3794 = x3794 + z3795 = x3795 + z3796 = x3796 + z3797 = x3797 + z3798 = x3798 + z3799 = x3799 + z3800 = x3800 + z3801 = x3801 + z3802 = x3802 + z3803 = x3803 + z3804 = x3804 + z3805 = x3805 + z3806 = x3806 + z3807 = x3807 + z3808 = x3808 + z3809 = x3809 + z3810 = x3810 + z3811 = x3811 + z3812 = x3812 + z3813 = x3813 + z3814 = x3814 + z3815 = x3815 + z3816 = x3816 + z3817 = x3817 + z3818 = x3818 + z3819 = x3819 + z3820 = x3820 + z3821 = x3821 + z3822 = x3822 + z3823 = x3823 + z3824 = x3824 + z3825 = x3825 + z3826 = x3826 + z3827 = x3827 + z3828 = x3828 + z3829 = x3829 + z3830 = x3830 + z3831 = x3831 + z3832 = x3832 + z3833 = x3833 + z3834 = x3834 + z3835 = x3835 + z3836 = x3836 + z3837 = x3837 + z3838 = x3838 + z3839 = x3839 + z3840 = x3840 + z3841 = x3841 + z3842 = x3842 + z3843 = x3843 + z3844 = x3844 + z3845 = x3845 + z3846 = x3846 + z3847 = x3847 + z3848 = x3848 + z3849 = x3849 + z3850 = x3850 + z3851 = x3851 + z3852 = x3852 + z3853 = x3853 + z3854 = x3854 + z3855 = x3855 + z3856 = x3856 + z3857 = x3857 + z3858 = x3858 + z3859 = x3859 + z3860 = x3860 + z3861 = x3861 + z3862 = x3862 + z3863 = x3863 + z3864 = x3864 + z3865 = x3865 + z3866 = x3866 + z3867 = x3867 + z3868 = x3868 + z3869 = x3869 + z3870 = x3870 + z3871 = x3871 + z3872 = x3872 + z3873 = x3873 + z3874 = x3874 + z3875 = x3875 + z3876 = x3876 + z3877 = x3877 + z3878 = x3878 + z3879 = x3879 + z3880 = x3880 + z3881 = x3881 + z3882 = x3882 + z3883 = x3883 + z3884 = x3884 + z3885 = x3885 + z3886 = x3886 + z3887 = x3887 + z3888 = x3888 + z3889 = x3889 + z3890 = x3890 + z3891 = x3891 + z3892 = x3892 + z3893 = x3893 + z3894 = x3894 + z3895 = x3895 + z3896 = x3896 + z3897 = x3897 + z3898 = x3898 + z3899 = x3899 + z3900 = x3900 + z3901 = x3901 + z3902 = x3902 + z3903 = x3903 + z3904 = x3904 + z3905 = x3905 + z3906 = x3906 + z3907 = x3907 + z3908 = x3908 + z3909 = x3909 + z3910 = x3910 + z3911 = x3911 + z3912 = x3912 + z3913 = x3913 + z3914 = x3914 + z3915 = x3915 + z3916 = x3916 + z3917 = x3917 + z3918 = x3918 + z3919 = x3919 + z3920 = x3920 + z3921 = x3921 + z3922 = x3922 + z3923 = x3923 + z3924 = x3924 + z3925 = x3925 + z3926 = x3926 + z3927 = x3927 + z3928 = x3928 + z3929 = x3929 + z3930 = x3930 + z3931 = x3931 + z3932 = x3932 + z3933 = x3933 + z3934 = x3934 + z3935 = x3935 + z3936 = x3936 + z3937 = x3937 + z3938 = x3938 + z3939 = x3939 + z3940 = x3940 + z3941 = x3941 + z3942 = x3942 + z3943 = x3943 + z3944 = x3944 + z3945 = x3945 + z3946 = x3946 + z3947 = x3947 + z3948 = x3948 + z3949 = x3949 + z3950 = x3950 + z3951 = x3951 + z3952 = x3952 + z3953 = x3953 + z3954 = x3954 + z3955 = x3955 + z3956 = x3956 + z3957 = x3957 + z3958 = x3958 + z3959 = x3959 + z3960 = x3960 + z3961 = x3961 + z3962 = x3962 + z3963 = x3963 + z3964 = x3964 + z3965 = x3965 + z3966 = x3966 + z3967 = x3967 + z3968 = x3968 + z3969 = x3969 + z3970 = x3970 + z3971 = x3971 + z3972 = x3972 + z3973 = x3973 + z3974 = x3974 + z3975 = x3975 + z3976 = x3976 + z3977 = x3977 + z3978 = x3978 + z3979 = x3979 + z3980 = x3980 + z3981 = x3981 + z3982 = x3982 + z3983 = x3983 + z3984 = x3984 + z3985 = x3985 + z3986 = x3986 + z3987 = x3987 + z3988 = x3988 + z3989 = x3989 + z3990 = x3990 + z3991 = x3991 + z3992 = x3992 + z3993 = x3993 + z3994 = x3994 + z3995 = x3995 + z3996 = x3996 + z3997 = x3997 + z3998 = x3998 + z3999 = x3999 + z4000 = x4000 + z4001 = x4001 + z4002 = x4002 + z4003 = x4003 + z4004 = x4004 + z4005 = x4005 + z4006 = x4006 + z4007 = x4007 + z4008 = x4008 + z4009 = x4009 + z4010 = x4010 + z4011 = x4011 + z4012 = x4012 + z4013 = x4013 + z4014 = x4014 + z4015 = x4015 + z4016 = x4016 + z4017 = x4017 + z4018 = x4018 + z4019 = x4019 + z4020 = x4020 + z4021 = x4021 + z4022 = x4022 + z4023 = x4023 + z4024 = x4024 + z4025 = x4025 + z4026 = x4026 + z4027 = x4027 + z4028 = x4028 + z4029 = x4029 + z4030 = x4030 + z4031 = x4031 + z4032 = x4032 + z4033 = x4033 + z4034 = x4034 + z4035 = x4035 + z4036 = x4036 + z4037 = x4037 + z4038 = x4038 + z4039 = x4039 + z4040 = x4040 + z4041 = x4041 + z4042 = x4042 + z4043 = x4043 + z4044 = x4044 + z4045 = x4045 + z4046 = x4046 + z4047 = x4047 + z4048 = x4048 + z4049 = x4049 + z4050 = x4050 + z4051 = x4051 + z4052 = x4052 + z4053 = x4053 + z4054 = x4054 + z4055 = x4055 + z4056 = x4056 + z4057 = x4057 + z4058 = x4058 + z4059 = x4059 + z4060 = x4060 + z4061 = x4061 + z4062 = x4062 + z4063 = x4063 + z4064 = x4064 + z4065 = x4065 + z4066 = x4066 + z4067 = x4067 + z4068 = x4068 + z4069 = x4069 + z4070 = x4070 + z4071 = x4071 + z4072 = x4072 + z4073 = x4073 + z4074 = x4074 + z4075 = x4075 + z4076 = x4076 + z4077 = x4077 + z4078 = x4078 + z4079 = x4079 + z4080 = x4080 + z4081 = x4081 + z4082 = x4082 + z4083 = x4083 + z4084 = x4084 + z4085 = x4085 + z4086 = x4086 + z4087 = x4087 + z4088 = x4088 + z4089 = x4089 + z4090 = x4090 + z4091 = x4091 + z4092 = x4092 + z4093 = x4093 + z4094 = x4094 + z4095 = x4095 + z4096 = x4096 + z4097 = x4097 + z4098 = x4098 + z4099 = x4099 + z4100 = x4100 + z4101 = x4101 + z4102 = x4102 + z4103 = x4103 + z4104 = x4104 + z4105 = x4105 + z4106 = x4106 + z4107 = x4107 + z4108 = x4108 + z4109 = x4109 + z4110 = x4110 + z4111 = x4111 + z4112 = x4112 + z4113 = x4113 + z4114 = x4114 + z4115 = x4115 + z4116 = x4116 + z4117 = x4117 + z4118 = x4118 + z4119 = x4119 + z4120 = x4120 + z4121 = x4121 + z4122 = x4122 + z4123 = x4123 + z4124 = x4124 + z4125 = x4125 + z4126 = x4126 + z4127 = x4127 + z4128 = x4128 + z4129 = x4129 + z4130 = x4130 + z4131 = x4131 + z4132 = x4132 + z4133 = x4133 + z4134 = x4134 + z4135 = x4135 + z4136 = x4136 + z4137 = x4137 + z4138 = x4138 + z4139 = x4139 + z4140 = x4140 + z4141 = x4141 + z4142 = x4142 + z4143 = x4143 + z4144 = x4144 + z4145 = x4145 + z4146 = x4146 + z4147 = x4147 + z4148 = x4148 + z4149 = x4149 + z4150 = x4150 + z4151 = x4151 + z4152 = x4152 + z4153 = x4153 + z4154 = x4154 + z4155 = x4155 + z4156 = x4156 + z4157 = x4157 + z4158 = x4158 + z4159 = x4159 + z4160 = x4160 + z4161 = x4161 + z4162 = x4162 + z4163 = x4163 + z4164 = x4164 + z4165 = x4165 + z4166 = x4166 + z4167 = x4167 + z4168 = x4168 + z4169 = x4169 + z4170 = x4170 + z4171 = x4171 + z4172 = x4172 + z4173 = x4173 + z4174 = x4174 + z4175 = x4175 + z4176 = x4176 + z4177 = x4177 + z4178 = x4178 + z4179 = x4179 + z4180 = x4180 + z4181 = x4181 + z4182 = x4182 + z4183 = x4183 + z4184 = x4184 + z4185 = x4185 + z4186 = x4186 + z4187 = x4187 + z4188 = x4188 + z4189 = x4189 + z4190 = x4190 + z4191 = x4191 + z4192 = x4192 + z4193 = x4193 + z4194 = x4194 + z4195 = x4195 + z4196 = x4196 + z4197 = x4197 + z4198 = x4198 + z4199 = x4199 + z4200 = x4200 + z4201 = x4201 + z4202 = x4202 + z4203 = x4203 + z4204 = x4204 + z4205 = x4205 + z4206 = x4206 + z4207 = x4207 + z4208 = x4208 + z4209 = x4209 + z4210 = x4210 + z4211 = x4211 + z4212 = x4212 + z4213 = x4213 + z4214 = x4214 + z4215 = x4215 + z4216 = x4216 + z4217 = x4217 + z4218 = x4218 + z4219 = x4219 + z4220 = x4220 + z4221 = x4221 + z4222 = x4222 + z4223 = x4223 + z4224 = x4224 + z4225 = x4225 + z4226 = x4226 + z4227 = x4227 + z4228 = x4228 + z4229 = x4229 + z4230 = x4230 + z4231 = x4231 + z4232 = x4232 + z4233 = x4233 + z4234 = x4234 + z4235 = x4235 + z4236 = x4236 + z4237 = x4237 + z4238 = x4238 + z4239 = x4239 + z4240 = x4240 + z4241 = x4241 + z4242 = x4242 + z4243 = x4243 + z4244 = x4244 + z4245 = x4245 + z4246 = x4246 + z4247 = x4247 + z4248 = x4248 + z4249 = x4249 + z4250 = x4250 + z4251 = x4251 + z4252 = x4252 + z4253 = x4253 + z4254 = x4254 + z4255 = x4255 + z4256 = x4256 + z4257 = x4257 + z4258 = x4258 + z4259 = x4259 + z4260 = x4260 + z4261 = x4261 + z4262 = x4262 + z4263 = x4263 + z4264 = x4264 + z4265 = x4265 + z4266 = x4266 + z4267 = x4267 + z4268 = x4268 + z4269 = x4269 + z4270 = x4270 + z4271 = x4271 + z4272 = x4272 + z4273 = x4273 + z4274 = x4274 + z4275 = x4275 + z4276 = x4276 + z4277 = x4277 + z4278 = x4278 + z4279 = x4279 + z4280 = x4280 + z4281 = x4281 + z4282 = x4282 + z4283 = x4283 + z4284 = x4284 + z4285 = x4285 + z4286 = x4286 + z4287 = x4287 + z4288 = x4288 + z4289 = x4289 + z4290 = x4290 + z4291 = x4291 + z4292 = x4292 + z4293 = x4293 + z4294 = x4294 + z4295 = x4295 + z4296 = x4296 + z4297 = x4297 + z4298 = x4298 + z4299 = x4299 + z4300 = x4300 + z4301 = x4301 + z4302 = x4302 + z4303 = x4303 + z4304 = x4304 + z4305 = x4305 + z4306 = x4306 + z4307 = x4307 + z4308 = x4308 + z4309 = x4309 + z4310 = x4310 + z4311 = x4311 + z4312 = x4312 + z4313 = x4313 + z4314 = x4314 + z4315 = x4315 + z4316 = x4316 + z4317 = x4317 + z4318 = x4318 + z4319 = x4319 + z4320 = x4320 + z4321 = x4321 + z4322 = x4322 + z4323 = x4323 + z4324 = x4324 + z4325 = x4325 + z4326 = x4326 + z4327 = x4327 + z4328 = x4328 + z4329 = x4329 + z4330 = x4330 + z4331 = x4331 + z4332 = x4332 + z4333 = x4333 + z4334 = x4334 + z4335 = x4335 + z4336 = x4336 + z4337 = x4337 + z4338 = x4338 + z4339 = x4339 + z4340 = x4340 + z4341 = x4341 + z4342 = x4342 + z4343 = x4343 + z4344 = x4344 + z4345 = x4345 + z4346 = x4346 + z4347 = x4347 + z4348 = x4348 + z4349 = x4349 + z4350 = x4350 + z4351 = x4351 + z4352 = x4352 + z4353 = x4353 + z4354 = x4354 + z4355 = x4355 + z4356 = x4356 + z4357 = x4357 + z4358 = x4358 + z4359 = x4359 + z4360 = x4360 + z4361 = x4361 + z4362 = x4362 + z4363 = x4363 + z4364 = x4364 + z4365 = x4365 + z4366 = x4366 + z4367 = x4367 + z4368 = x4368 + z4369 = x4369 + z4370 = x4370 + z4371 = x4371 + z4372 = x4372 + z4373 = x4373 + z4374 = x4374 + z4375 = x4375 + z4376 = x4376 + z4377 = x4377 + z4378 = x4378 + z4379 = x4379 + z4380 = x4380 + z4381 = x4381 + z4382 = x4382 + z4383 = x4383 + z4384 = x4384 + z4385 = x4385 + z4386 = x4386 + z4387 = x4387 + z4388 = x4388 + z4389 = x4389 + z4390 = x4390 + z4391 = x4391 + z4392 = x4392 + z4393 = x4393 + z4394 = x4394 + z4395 = x4395 + z4396 = x4396 + z4397 = x4397 + z4398 = x4398 + z4399 = x4399 + z4400 = x4400 + z4401 = x4401 + z4402 = x4402 + z4403 = x4403 + z4404 = x4404 + z4405 = x4405 + z4406 = x4406 + z4407 = x4407 + z4408 = x4408 + z4409 = x4409 + z4410 = x4410 + z4411 = x4411 + z4412 = x4412 + z4413 = x4413 + z4414 = x4414 + z4415 = x4415 + z4416 = x4416 + z4417 = x4417 + z4418 = x4418 + z4419 = x4419 + z4420 = x4420 + z4421 = x4421 + z4422 = x4422 + z4423 = x4423 + z4424 = x4424 + z4425 = x4425 + z4426 = x4426 + z4427 = x4427 + z4428 = x4428 + z4429 = x4429 + z4430 = x4430 + z4431 = x4431 + z4432 = x4432 + z4433 = x4433 + z4434 = x4434 + z4435 = x4435 + z4436 = x4436 + z4437 = x4437 + z4438 = x4438 + z4439 = x4439 + z4440 = x4440 + z4441 = x4441 + z4442 = x4442 + z4443 = x4443 + z4444 = x4444 + z4445 = x4445 + z4446 = x4446 + z4447 = x4447 + z4448 = x4448 + z4449 = x4449 + z4450 = x4450 + z4451 = x4451 + z4452 = x4452 + z4453 = x4453 + z4454 = x4454 + z4455 = x4455 + z4456 = x4456 + z4457 = x4457 + z4458 = x4458 + z4459 = x4459 + z4460 = x4460 + z4461 = x4461 + z4462 = x4462 + z4463 = x4463 + z4464 = x4464 + z4465 = x4465 + z4466 = x4466 + z4467 = x4467 + z4468 = x4468 + z4469 = x4469 + z4470 = x4470 + z4471 = x4471 + z4472 = x4472 + z4473 = x4473 + z4474 = x4474 + z4475 = x4475 + z4476 = x4476 + z4477 = x4477 + z4478 = x4478 + z4479 = x4479 + z4480 = x4480 + z4481 = x4481 + z4482 = x4482 + z4483 = x4483 + z4484 = x4484 + z4485 = x4485 + z4486 = x4486 + z4487 = x4487 + z4488 = x4488 + z4489 = x4489 + z4490 = x4490 + z4491 = x4491 + z4492 = x4492 + z4493 = x4493 + z4494 = x4494 + z4495 = x4495 + z4496 = x4496 + z4497 = x4497 + z4498 = x4498 + z4499 = x4499 + z4500 = x4500 + z4501 = x4501 + z4502 = x4502 + z4503 = x4503 + z4504 = x4504 + z4505 = x4505 + z4506 = x4506 + z4507 = x4507 + z4508 = x4508 + z4509 = x4509 + z4510 = x4510 + z4511 = x4511 + z4512 = x4512 + z4513 = x4513 + z4514 = x4514 + z4515 = x4515 + z4516 = x4516 + z4517 = x4517 + z4518 = x4518 + z4519 = x4519 + z4520 = x4520 + z4521 = x4521 + z4522 = x4522 + z4523 = x4523 + z4524 = x4524 + z4525 = x4525 + z4526 = x4526 + z4527 = x4527 + z4528 = x4528 + z4529 = x4529 + z4530 = x4530 + z4531 = x4531 + z4532 = x4532 + z4533 = x4533 + z4534 = x4534 + z4535 = x4535 + z4536 = x4536 + z4537 = x4537 + z4538 = x4538 + z4539 = x4539 + z4540 = x4540 + z4541 = x4541 + z4542 = x4542 + z4543 = x4543 + z4544 = x4544 + z4545 = x4545 + z4546 = x4546 + z4547 = x4547 + z4548 = x4548 + z4549 = x4549 + z4550 = x4550 + z4551 = x4551 + z4552 = x4552 + z4553 = x4553 + z4554 = x4554 + z4555 = x4555 + z4556 = x4556 + z4557 = x4557 + z4558 = x4558 + z4559 = x4559 + z4560 = x4560 + z4561 = x4561 + z4562 = x4562 + z4563 = x4563 + z4564 = x4564 + z4565 = x4565 + z4566 = x4566 + z4567 = x4567 + z4568 = x4568 + z4569 = x4569 + z4570 = x4570 + z4571 = x4571 + z4572 = x4572 + z4573 = x4573 + z4574 = x4574 + z4575 = x4575 + z4576 = x4576 + z4577 = x4577 + z4578 = x4578 + z4579 = x4579 + z4580 = x4580 + z4581 = x4581 + z4582 = x4582 + z4583 = x4583 + z4584 = x4584 + z4585 = x4585 + z4586 = x4586 + z4587 = x4587 + z4588 = x4588 + z4589 = x4589 + z4590 = x4590 + z4591 = x4591 + z4592 = x4592 + z4593 = x4593 + z4594 = x4594 + z4595 = x4595 + z4596 = x4596 + z4597 = x4597 + z4598 = x4598 + z4599 = x4599 + z4600 = x4600 + z4601 = x4601 + z4602 = x4602 + z4603 = x4603 + z4604 = x4604 + z4605 = x4605 + z4606 = x4606 + z4607 = x4607 + z4608 = x4608 + z4609 = x4609 + z4610 = x4610 + z4611 = x4611 + z4612 = x4612 + z4613 = x4613 + z4614 = x4614 + z4615 = x4615 + z4616 = x4616 + z4617 = x4617 + z4618 = x4618 + z4619 = x4619 + z4620 = x4620 + z4621 = x4621 + z4622 = x4622 + z4623 = x4623 + z4624 = x4624 + z4625 = x4625 + z4626 = x4626 + z4627 = x4627 + z4628 = x4628 + z4629 = x4629 + z4630 = x4630 + z4631 = x4631 + z4632 = x4632 + z4633 = x4633 + z4634 = x4634 + z4635 = x4635 + z4636 = x4636 + z4637 = x4637 + z4638 = x4638 + z4639 = x4639 + z4640 = x4640 + z4641 = x4641 + z4642 = x4642 + z4643 = x4643 + z4644 = x4644 + z4645 = x4645 + z4646 = x4646 + z4647 = x4647 + z4648 = x4648 + z4649 = x4649 + z4650 = x4650 + z4651 = x4651 + z4652 = x4652 + z4653 = x4653 + z4654 = x4654 + z4655 = x4655 + z4656 = x4656 + z4657 = x4657 + z4658 = x4658 + z4659 = x4659 + z4660 = x4660 + z4661 = x4661 + z4662 = x4662 + z4663 = x4663 + z4664 = x4664 + z4665 = x4665 + z4666 = x4666 + z4667 = x4667 + z4668 = x4668 + z4669 = x4669 + z4670 = x4670 + z4671 = x4671 + z4672 = x4672 + z4673 = x4673 + z4674 = x4674 + z4675 = x4675 + z4676 = x4676 + z4677 = x4677 + z4678 = x4678 + z4679 = x4679 + z4680 = x4680 + z4681 = x4681 + z4682 = x4682 + z4683 = x4683 + z4684 = x4684 + z4685 = x4685 + z4686 = x4686 + z4687 = x4687 + z4688 = x4688 + z4689 = x4689 + z4690 = x4690 + z4691 = x4691 + z4692 = x4692 + z4693 = x4693 + z4694 = x4694 + z4695 = x4695 + z4696 = x4696 + z4697 = x4697 + z4698 = x4698 + z4699 = x4699 + z4700 = x4700 + z4701 = x4701 + z4702 = x4702 + z4703 = x4703 + z4704 = x4704 + z4705 = x4705 + z4706 = x4706 + z4707 = x4707 + z4708 = x4708 + z4709 = x4709 + z4710 = x4710 + z4711 = x4711 + z4712 = x4712 + z4713 = x4713 + z4714 = x4714 + z4715 = x4715 + z4716 = x4716 + z4717 = x4717 + z4718 = x4718 + z4719 = x4719 + z4720 = x4720 + z4721 = x4721 + z4722 = x4722 + z4723 = x4723 + z4724 = x4724 + z4725 = x4725 + z4726 = x4726 + z4727 = x4727 + z4728 = x4728 + z4729 = x4729 + z4730 = x4730 + z4731 = x4731 + z4732 = x4732 + z4733 = x4733 + z4734 = x4734 + z4735 = x4735 + z4736 = x4736 + z4737 = x4737 + z4738 = x4738 + z4739 = x4739 + z4740 = x4740 + z4741 = x4741 + z4742 = x4742 + z4743 = x4743 + z4744 = x4744 + z4745 = x4745 + z4746 = x4746 + z4747 = x4747 + z4748 = x4748 + z4749 = x4749 + z4750 = x4750 + z4751 = x4751 + z4752 = x4752 + z4753 = x4753 + z4754 = x4754 + z4755 = x4755 + z4756 = x4756 + z4757 = x4757 + z4758 = x4758 + z4759 = x4759 + z4760 = x4760 + z4761 = x4761 + z4762 = x4762 + z4763 = x4763 + z4764 = x4764 + z4765 = x4765 + z4766 = x4766 + z4767 = x4767 + z4768 = x4768 + z4769 = x4769 + z4770 = x4770 + z4771 = x4771 + z4772 = x4772 + z4773 = x4773 + z4774 = x4774 + z4775 = x4775 + z4776 = x4776 + z4777 = x4777 + z4778 = x4778 + z4779 = x4779 + z4780 = x4780 + z4781 = x4781 + z4782 = x4782 + z4783 = x4783 + z4784 = x4784 + z4785 = x4785 + z4786 = x4786 + z4787 = x4787 + z4788 = x4788 + z4789 = x4789 + z4790 = x4790 + z4791 = x4791 + z4792 = x4792 + z4793 = x4793 + z4794 = x4794 + z4795 = x4795 + z4796 = x4796 + z4797 = x4797 + z4798 = x4798 + z4799 = x4799 + z4800 = x4800 + z4801 = x4801 + z4802 = x4802 + z4803 = x4803 + z4804 = x4804 + z4805 = x4805 + z4806 = x4806 + z4807 = x4807 + z4808 = x4808 + z4809 = x4809 + z4810 = x4810 + z4811 = x4811 + z4812 = x4812 + z4813 = x4813 + z4814 = x4814 + z4815 = x4815 + z4816 = x4816 + z4817 = x4817 + z4818 = x4818 + z4819 = x4819 + z4820 = x4820 + z4821 = x4821 + z4822 = x4822 + z4823 = x4823 + z4824 = x4824 + z4825 = x4825 + z4826 = x4826 + z4827 = x4827 + z4828 = x4828 + z4829 = x4829 + z4830 = x4830 + z4831 = x4831 + z4832 = x4832 + z4833 = x4833 + z4834 = x4834 + z4835 = x4835 + z4836 = x4836 + z4837 = x4837 + z4838 = x4838 + z4839 = x4839 + z4840 = x4840 + z4841 = x4841 + z4842 = x4842 + z4843 = x4843 + z4844 = x4844 + z4845 = x4845 + z4846 = x4846 + z4847 = x4847 + z4848 = x4848 + z4849 = x4849 + z4850 = x4850 + z4851 = x4851 + z4852 = x4852 + z4853 = x4853 + z4854 = x4854 + z4855 = x4855 + z4856 = x4856 + z4857 = x4857 + z4858 = x4858 + z4859 = x4859 + z4860 = x4860 + z4861 = x4861 + z4862 = x4862 + z4863 = x4863 + z4864 = x4864 + z4865 = x4865 + z4866 = x4866 + z4867 = x4867 + z4868 = x4868 + z4869 = x4869 + z4870 = x4870 + z4871 = x4871 + z4872 = x4872 + z4873 = x4873 + z4874 = x4874 + z4875 = x4875 + z4876 = x4876 + z4877 = x4877 + z4878 = x4878 + z4879 = x4879 + z4880 = x4880 + z4881 = x4881 + z4882 = x4882 + z4883 = x4883 + z4884 = x4884 + z4885 = x4885 + z4886 = x4886 + z4887 = x4887 + z4888 = x4888 + z4889 = x4889 + z4890 = x4890 + z4891 = x4891 + z4892 = x4892 + z4893 = x4893 + z4894 = x4894 + z4895 = x4895 + z4896 = x4896 + z4897 = x4897 + z4898 = x4898 + z4899 = x4899 + z4900 = x4900 + z4901 = x4901 + z4902 = x4902 + z4903 = x4903 + z4904 = x4904 + z4905 = x4905 + z4906 = x4906 + z4907 = x4907 + z4908 = x4908 + z4909 = x4909 + z4910 = x4910 + z4911 = x4911 + z4912 = x4912 + z4913 = x4913 + z4914 = x4914 + z4915 = x4915 + z4916 = x4916 + z4917 = x4917 + z4918 = x4918 + z4919 = x4919 + z4920 = x4920 + z4921 = x4921 + z4922 = x4922 + z4923 = x4923 + z4924 = x4924 + z4925 = x4925 + z4926 = x4926 + z4927 = x4927 + z4928 = x4928 + z4929 = x4929 + z4930 = x4930 + z4931 = x4931 + z4932 = x4932 + z4933 = x4933 + z4934 = x4934 + z4935 = x4935 + z4936 = x4936 + z4937 = x4937 + z4938 = x4938 + z4939 = x4939 + z4940 = x4940 + z4941 = x4941 + z4942 = x4942 + z4943 = x4943 + z4944 = x4944 + z4945 = x4945 + z4946 = x4946 + z4947 = x4947 + z4948 = x4948 + z4949 = x4949 + z4950 = x4950 + z4951 = x4951 + z4952 = x4952 + z4953 = x4953 + z4954 = x4954 + z4955 = x4955 + z4956 = x4956 + z4957 = x4957 + z4958 = x4958 + z4959 = x4959 + z4960 = x4960 + z4961 = x4961 + z4962 = x4962 + z4963 = x4963 + z4964 = x4964 + z4965 = x4965 + z4966 = x4966 + z4967 = x4967 + z4968 = x4968 + z4969 = x4969 + z4970 = x4970 + z4971 = x4971 + z4972 = x4972 + z4973 = x4973 + z4974 = x4974 + z4975 = x4975 + z4976 = x4976 + z4977 = x4977 + z4978 = x4978 + z4979 = x4979 + z4980 = x4980 + z4981 = x4981 + z4982 = x4982 + z4983 = x4983 + z4984 = x4984 + z4985 = x4985 + z4986 = x4986 + z4987 = x4987 + z4988 = x4988 + z4989 = x4989 + z4990 = x4990 + z4991 = x4991 + z4992 = x4992 + z4993 = x4993 + z4994 = x4994 + z4995 = x4995 + z4996 = x4996 + z4997 = x4997 + z4998 = x4998 + z4999 = x4999 + z5000 = x5000 + z5001 = x5001 + z5002 = x5002 + z5003 = x5003 + z5004 = x5004 + z5005 = x5005 + z5006 = x5006 + z5007 = x5007 + z5008 = x5008 + z5009 = x5009 + z5010 = x5010 + z5011 = x5011 + z5012 = x5012 + z5013 = x5013 + z5014 = x5014 + z5015 = x5015 + z5016 = x5016 + z5017 = x5017 + z5018 = x5018 + z5019 = x5019 + z5020 = x5020 + z5021 = x5021 + z5022 = x5022 + z5023 = x5023 + z5024 = x5024 + z5025 = x5025 + z5026 = x5026 + z5027 = x5027 + z5028 = x5028 + z5029 = x5029 + z5030 = x5030 + z5031 = x5031 + z5032 = x5032 + z5033 = x5033 + z5034 = x5034 + z5035 = x5035 + z5036 = x5036 + z5037 = x5037 + z5038 = x5038 + z5039 = x5039 + z5040 = x5040 + z5041 = x5041 + z5042 = x5042 + z5043 = x5043 + z5044 = x5044 + z5045 = x5045 + z5046 = x5046 + z5047 = x5047 + z5048 = x5048 + z5049 = x5049 + z5050 = x5050 + z5051 = x5051 + z5052 = x5052 + z5053 = x5053 + z5054 = x5054 + z5055 = x5055 + z5056 = x5056 + z5057 = x5057 + z5058 = x5058 + z5059 = x5059 + z5060 = x5060 + z5061 = x5061 + z5062 = x5062 + z5063 = x5063 + z5064 = x5064 + z5065 = x5065 + z5066 = x5066 + z5067 = x5067 + z5068 = x5068 + z5069 = x5069 + z5070 = x5070 + z5071 = x5071 + z5072 = x5072 + z5073 = x5073 + z5074 = x5074 + z5075 = x5075 + z5076 = x5076 + z5077 = x5077 + z5078 = x5078 + z5079 = x5079 + z5080 = x5080 + z5081 = x5081 + z5082 = x5082 + z5083 = x5083 + z5084 = x5084 + z5085 = x5085 + z5086 = x5086 + z5087 = x5087 + z5088 = x5088 + z5089 = x5089 + z5090 = x5090 + z5091 = x5091 + z5092 = x5092 + z5093 = x5093 + z5094 = x5094 + z5095 = x5095 + z5096 = x5096 + z5097 = x5097 + z5098 = x5098 + z5099 = x5099 + z5100 = x5100 + z5101 = x5101 + z5102 = x5102 + z5103 = x5103 + z5104 = x5104 + z5105 = x5105 + z5106 = x5106 + z5107 = x5107 + z5108 = x5108 + z5109 = x5109 + z5110 = x5110 + z5111 = x5111 + z5112 = x5112 + z5113 = x5113 + z5114 = x5114 + z5115 = x5115 + z5116 = x5116 + z5117 = x5117 + z5118 = x5118 + z5119 = x5119 + z5120 = x5120 + z5121 = x5121 + z5122 = x5122 + z5123 = x5123 + z5124 = x5124 + z5125 = x5125 + z5126 = x5126 + z5127 = x5127 + z5128 = x5128 + z5129 = x5129 + z5130 = x5130 + z5131 = x5131 + z5132 = x5132 + z5133 = x5133 + z5134 = x5134 + z5135 = x5135 + z5136 = x5136 + z5137 = x5137 + z5138 = x5138 + z5139 = x5139 + z5140 = x5140 + z5141 = x5141 + z5142 = x5142 + z5143 = x5143 + z5144 = x5144 + z5145 = x5145 + z5146 = x5146 + z5147 = x5147 + z5148 = x5148 + z5149 = x5149 + z5150 = x5150 + z5151 = x5151 + z5152 = x5152 + z5153 = x5153 + z5154 = x5154 + z5155 = x5155 + z5156 = x5156 + z5157 = x5157 + z5158 = x5158 + z5159 = x5159 + z5160 = x5160 + z5161 = x5161 + z5162 = x5162 + z5163 = x5163 + z5164 = x5164 + z5165 = x5165 + z5166 = x5166 + z5167 = x5167 + z5168 = x5168 + z5169 = x5169 + z5170 = x5170 + z5171 = x5171 + z5172 = x5172 + z5173 = x5173 + z5174 = x5174 + z5175 = x5175 + z5176 = x5176 + z5177 = x5177 + z5178 = x5178 + z5179 = x5179 + z5180 = x5180 + z5181 = x5181 + z5182 = x5182 + z5183 = x5183 + z5184 = x5184 + z5185 = x5185 + z5186 = x5186 + z5187 = x5187 + z5188 = x5188 + z5189 = x5189 + z5190 = x5190 + z5191 = x5191 + z5192 = x5192 + z5193 = x5193 + z5194 = x5194 + z5195 = x5195 + z5196 = x5196 + z5197 = x5197 + z5198 = x5198 + z5199 = x5199 + z5200 = x5200 + z5201 = x5201 + z5202 = x5202 + z5203 = x5203 + z5204 = x5204 + z5205 = x5205 + z5206 = x5206 + z5207 = x5207 + z5208 = x5208 + z5209 = x5209 + z5210 = x5210 + z5211 = x5211 + z5212 = x5212 + z5213 = x5213 + z5214 = x5214 + z5215 = x5215 + z5216 = x5216 + z5217 = x5217 + z5218 = x5218 + z5219 = x5219 + z5220 = x5220 + z5221 = x5221 + z5222 = x5222 + z5223 = x5223 + z5224 = x5224 + z5225 = x5225 + z5226 = x5226 + z5227 = x5227 + z5228 = x5228 + z5229 = x5229 + z5230 = x5230 + z5231 = x5231 + z5232 = x5232 + z5233 = x5233 + z5234 = x5234 + z5235 = x5235 + z5236 = x5236 + z5237 = x5237 + z5238 = x5238 + z5239 = x5239 + z5240 = x5240 + z5241 = x5241 + z5242 = x5242 + z5243 = x5243 + z5244 = x5244 + z5245 = x5245 + z5246 = x5246 + z5247 = x5247 + z5248 = x5248 + z5249 = x5249 + z5250 = x5250 + z5251 = x5251 + z5252 = x5252 + z5253 = x5253 + z5254 = x5254 + z5255 = x5255 + z5256 = x5256 + z5257 = x5257 + z5258 = x5258 + z5259 = x5259 + z5260 = x5260 + z5261 = x5261 + z5262 = x5262 + z5263 = x5263 + z5264 = x5264 + z5265 = x5265 + z5266 = x5266 + z5267 = x5267 + z5268 = x5268 + z5269 = x5269 + z5270 = x5270 + z5271 = x5271 + z5272 = x5272 + z5273 = x5273 + z5274 = x5274 + z5275 = x5275 + z5276 = x5276 + z5277 = x5277 + z5278 = x5278 + z5279 = x5279 + z5280 = x5280 + z5281 = x5281 + z5282 = x5282 + z5283 = x5283 + z5284 = x5284 + z5285 = x5285 + z5286 = x5286 + z5287 = x5287 + z5288 = x5288 + z5289 = x5289 + z5290 = x5290 + z5291 = x5291 + z5292 = x5292 + z5293 = x5293 + z5294 = x5294 + z5295 = x5295 + z5296 = x5296 + z5297 = x5297 + z5298 = x5298 + z5299 = x5299 + z5300 = x5300 + z5301 = x5301 + z5302 = x5302 + z5303 = x5303 + z5304 = x5304 + z5305 = x5305 + z5306 = x5306 + z5307 = x5307 + z5308 = x5308 + z5309 = x5309 + z5310 = x5310 + z5311 = x5311 + z5312 = x5312 + z5313 = x5313 + z5314 = x5314 + z5315 = x5315 + z5316 = x5316 + z5317 = x5317 + z5318 = x5318 + z5319 = x5319 + z5320 = x5320 + z5321 = x5321 + z5322 = x5322 + z5323 = x5323 + z5324 = x5324 + z5325 = x5325 + z5326 = x5326 + z5327 = x5327 + z5328 = x5328 + z5329 = x5329 + z5330 = x5330 + z5331 = x5331 + z5332 = x5332 + z5333 = x5333 + z5334 = x5334 + z5335 = x5335 + z5336 = x5336 + z5337 = x5337 + z5338 = x5338 + z5339 = x5339 + z5340 = x5340 + z5341 = x5341 + z5342 = x5342 + z5343 = x5343 + z5344 = x5344 + z5345 = x5345 + z5346 = x5346 + z5347 = x5347 + z5348 = x5348 + z5349 = x5349 + z5350 = x5350 + z5351 = x5351 + z5352 = x5352 + z5353 = x5353 + z5354 = x5354 + z5355 = x5355 + z5356 = x5356 + z5357 = x5357 + z5358 = x5358 + z5359 = x5359 + z5360 = x5360 + z5361 = x5361 + z5362 = x5362 + z5363 = x5363 + z5364 = x5364 + z5365 = x5365 + z5366 = x5366 + z5367 = x5367 + z5368 = x5368 + z5369 = x5369 + z5370 = x5370 + z5371 = x5371 + z5372 = x5372 + z5373 = x5373 + z5374 = x5374 + z5375 = x5375 + z5376 = x5376 + z5377 = x5377 + z5378 = x5378 + z5379 = x5379 + z5380 = x5380 + z5381 = x5381 + z5382 = x5382 + z5383 = x5383 + z5384 = x5384 + z5385 = x5385 + z5386 = x5386 + z5387 = x5387 + z5388 = x5388 + z5389 = x5389 + z5390 = x5390 + z5391 = x5391 + z5392 = x5392 + z5393 = x5393 + z5394 = x5394 + z5395 = x5395 + z5396 = x5396 + z5397 = x5397 + z5398 = x5398 + z5399 = x5399 + z5400 = x5400 + z5401 = x5401 + z5402 = x5402 + z5403 = x5403 + z5404 = x5404 + z5405 = x5405 + z5406 = x5406 + z5407 = x5407 + z5408 = x5408 + z5409 = x5409 + z5410 = x5410 + z5411 = x5411 + z5412 = x5412 + z5413 = x5413 + z5414 = x5414 + z5415 = x5415 + z5416 = x5416 + z5417 = x5417 + z5418 = x5418 + z5419 = x5419 + z5420 = x5420 + z5421 = x5421 + z5422 = x5422 + z5423 = x5423 + z5424 = x5424 + z5425 = x5425 + z5426 = x5426 + z5427 = x5427 + z5428 = x5428 + z5429 = x5429 + z5430 = x5430 + z5431 = x5431 + z5432 = x5432 + z5433 = x5433 + z5434 = x5434 + z5435 = x5435 + z5436 = x5436 + z5437 = x5437 + z5438 = x5438 + z5439 = x5439 + z5440 = x5440 + z5441 = x5441 + z5442 = x5442 + z5443 = x5443 + z5444 = x5444 + z5445 = x5445 + z5446 = x5446 + z5447 = x5447 + z5448 = x5448 + z5449 = x5449 + z5450 = x5450 + z5451 = x5451 + z5452 = x5452 + z5453 = x5453 + z5454 = x5454 + z5455 = x5455 + z5456 = x5456 + z5457 = x5457 + z5458 = x5458 + z5459 = x5459 + z5460 = x5460 + z5461 = x5461 + z5462 = x5462 + z5463 = x5463 + z5464 = x5464 + z5465 = x5465 + z5466 = x5466 + z5467 = x5467 + z5468 = x5468 + z5469 = x5469 + z5470 = x5470 + z5471 = x5471 + z5472 = x5472 + z5473 = x5473 + z5474 = x5474 + z5475 = x5475 + z5476 = x5476 + z5477 = x5477 + z5478 = x5478 + z5479 = x5479 + z5480 = x5480 + z5481 = x5481 + z5482 = x5482 + z5483 = x5483 + z5484 = x5484 + z5485 = x5485 + z5486 = x5486 + z5487 = x5487 + z5488 = x5488 + z5489 = x5489 + z5490 = x5490 + z5491 = x5491 + z5492 = x5492 + z5493 = x5493 + z5494 = x5494 + z5495 = x5495 + z5496 = x5496 + z5497 = x5497 + z5498 = x5498 + z5499 = x5499 + z5500 = x5500 + z5501 = x5501 + z5502 = x5502 + z5503 = x5503 + z5504 = x5504 + z5505 = x5505 + z5506 = x5506 + z5507 = x5507 + z5508 = x5508 + z5509 = x5509 + z5510 = x5510 + z5511 = x5511 + z5512 = x5512 + z5513 = x5513 + z5514 = x5514 + z5515 = x5515 + z5516 = x5516 + z5517 = x5517 + z5518 = x5518 + z5519 = x5519 + z5520 = x5520 + z5521 = x5521 + z5522 = x5522 + z5523 = x5523 + z5524 = x5524 + z5525 = x5525 + z5526 = x5526 + z5527 = x5527 + z5528 = x5528 + z5529 = x5529 + z5530 = x5530 + z5531 = x5531 + z5532 = x5532 + z5533 = x5533 + z5534 = x5534 + z5535 = x5535 + z5536 = x5536 + z5537 = x5537 + z5538 = x5538 + z5539 = x5539 + z5540 = x5540 + z5541 = x5541 + z5542 = x5542 + z5543 = x5543 + z5544 = x5544 + z5545 = x5545 + z5546 = x5546 + z5547 = x5547 + z5548 = x5548 + z5549 = x5549 + z5550 = x5550 + z5551 = x5551 + z5552 = x5552 + z5553 = x5553 + z5554 = x5554 + z5555 = x5555 + z5556 = x5556 + z5557 = x5557 + z5558 = x5558 + z5559 = x5559 + z5560 = x5560 + z5561 = x5561 + z5562 = x5562 + z5563 = x5563 + z5564 = x5564 + z5565 = x5565 + z5566 = x5566 + z5567 = x5567 + z5568 = x5568 + z5569 = x5569 + z5570 = x5570 + z5571 = x5571 + z5572 = x5572 + z5573 = x5573 + z5574 = x5574 + z5575 = x5575 + z5576 = x5576 + z5577 = x5577 + z5578 = x5578 + z5579 = x5579 + z5580 = x5580 + z5581 = x5581 + z5582 = x5582 + z5583 = x5583 + z5584 = x5584 + z5585 = x5585 + z5586 = x5586 + z5587 = x5587 + z5588 = x5588 + z5589 = x5589 + z5590 = x5590 + z5591 = x5591 + z5592 = x5592 + z5593 = x5593 + z5594 = x5594 + z5595 = x5595 + z5596 = x5596 + z5597 = x5597 + z5598 = x5598 + z5599 = x5599 + z5600 = x5600 + z5601 = x5601 + z5602 = x5602 + z5603 = x5603 + z5604 = x5604 + z5605 = x5605 + z5606 = x5606 + z5607 = x5607 + z5608 = x5608 + z5609 = x5609 + z5610 = x5610 + z5611 = x5611 + z5612 = x5612 + z5613 = x5613 + z5614 = x5614 + z5615 = x5615 + z5616 = x5616 + z5617 = x5617 + z5618 = x5618 + z5619 = x5619 + z5620 = x5620 + z5621 = x5621 + z5622 = x5622 + z5623 = x5623 + z5624 = x5624 + z5625 = x5625 + z5626 = x5626 + z5627 = x5627 + z5628 = x5628 + z5629 = x5629 + z5630 = x5630 + z5631 = x5631 + z5632 = x5632 + z5633 = x5633 + z5634 = x5634 + z5635 = x5635 + z5636 = x5636 + z5637 = x5637 + z5638 = x5638 + z5639 = x5639 + z5640 = x5640 + z5641 = x5641 + z5642 = x5642 + z5643 = x5643 + z5644 = x5644 + z5645 = x5645 + z5646 = x5646 + z5647 = x5647 + z5648 = x5648 + z5649 = x5649 + z5650 = x5650 + z5651 = x5651 + z5652 = x5652 + z5653 = x5653 + z5654 = x5654 + z5655 = x5655 + z5656 = x5656 + z5657 = x5657 + z5658 = x5658 + z5659 = x5659 + z5660 = x5660 + z5661 = x5661 + z5662 = x5662 + z5663 = x5663 + z5664 = x5664 + z5665 = x5665 + z5666 = x5666 + z5667 = x5667 + z5668 = x5668 + z5669 = x5669 + z5670 = x5670 + z5671 = x5671 + z5672 = x5672 + z5673 = x5673 + z5674 = x5674 + z5675 = x5675 + z5676 = x5676 + z5677 = x5677 + z5678 = x5678 + z5679 = x5679 + z5680 = x5680 + z5681 = x5681 + z5682 = x5682 + z5683 = x5683 + z5684 = x5684 + z5685 = x5685 + z5686 = x5686 + z5687 = x5687 + z5688 = x5688 + z5689 = x5689 + z5690 = x5690 + z5691 = x5691 + z5692 = x5692 + z5693 = x5693 + z5694 = x5694 + z5695 = x5695 + z5696 = x5696 + z5697 = x5697 + z5698 = x5698 + z5699 = x5699 + z5700 = x5700 + z5701 = x5701 + z5702 = x5702 + z5703 = x5703 + z5704 = x5704 + z5705 = x5705 + z5706 = x5706 + z5707 = x5707 + z5708 = x5708 + z5709 = x5709 + z5710 = x5710 + z5711 = x5711 + z5712 = x5712 + z5713 = x5713 + z5714 = x5714 + z5715 = x5715 + z5716 = x5716 + z5717 = x5717 + z5718 = x5718 + z5719 = x5719 + z5720 = x5720 + z5721 = x5721 + z5722 = x5722 + z5723 = x5723 + z5724 = x5724 + z5725 = x5725 + z5726 = x5726 + z5727 = x5727 + z5728 = x5728 + z5729 = x5729 + z5730 = x5730 + z5731 = x5731 + z5732 = x5732 + z5733 = x5733 + z5734 = x5734 + z5735 = x5735 + z5736 = x5736 + z5737 = x5737 + z5738 = x5738 + z5739 = x5739 + z5740 = x5740 + z5741 = x5741 + z5742 = x5742 + z5743 = x5743 + z5744 = x5744 + z5745 = x5745 + z5746 = x5746 + z5747 = x5747 + z5748 = x5748 + z5749 = x5749 + z5750 = x5750 + z5751 = x5751 + z5752 = x5752 + z5753 = x5753 + z5754 = x5754 + z5755 = x5755 + z5756 = x5756 + z5757 = x5757 + z5758 = x5758 + z5759 = x5759 + z5760 = x5760 + z5761 = x5761 + z5762 = x5762 + z5763 = x5763 + z5764 = x5764 + z5765 = x5765 + z5766 = x5766 + z5767 = x5767 + z5768 = x5768 + z5769 = x5769 + z5770 = x5770 + z5771 = x5771 + z5772 = x5772 + z5773 = x5773 + z5774 = x5774 + z5775 = x5775 + z5776 = x5776 + z5777 = x5777 + z5778 = x5778 + z5779 = x5779 + z5780 = x5780 + z5781 = x5781 + z5782 = x5782 + z5783 = x5783 + z5784 = x5784 + z5785 = x5785 + z5786 = x5786 + z5787 = x5787 + z5788 = x5788 + z5789 = x5789 + z5790 = x5790 + z5791 = x5791 + z5792 = x5792 + z5793 = x5793 + z5794 = x5794 + z5795 = x5795 + z5796 = x5796 + z5797 = x5797 + z5798 = x5798 + z5799 = x5799 + z5800 = x5800 + z5801 = x5801 + z5802 = x5802 + z5803 = x5803 + z5804 = x5804 + z5805 = x5805 + z5806 = x5806 + z5807 = x5807 + z5808 = x5808 + z5809 = x5809 + z5810 = x5810 + z5811 = x5811 + z5812 = x5812 + z5813 = x5813 + z5814 = x5814 + z5815 = x5815 + z5816 = x5816 + z5817 = x5817 + z5818 = x5818 + z5819 = x5819 + z5820 = x5820 + z5821 = x5821 + z5822 = x5822 + z5823 = x5823 + z5824 = x5824 + z5825 = x5825 + z5826 = x5826 + z5827 = x5827 + z5828 = x5828 + z5829 = x5829 + z5830 = x5830 + z5831 = x5831 + z5832 = x5832 + z5833 = x5833 + z5834 = x5834 + z5835 = x5835 + z5836 = x5836 + z5837 = x5837 + z5838 = x5838 + z5839 = x5839 + z5840 = x5840 + z5841 = x5841 + z5842 = x5842 + z5843 = x5843 + z5844 = x5844 + z5845 = x5845 + z5846 = x5846 + z5847 = x5847 + z5848 = x5848 + z5849 = x5849 + z5850 = x5850 + z5851 = x5851 + z5852 = x5852 + z5853 = x5853 + z5854 = x5854 + z5855 = x5855 + z5856 = x5856 + z5857 = x5857 + z5858 = x5858 + z5859 = x5859 + z5860 = x5860 + z5861 = x5861 + z5862 = x5862 + z5863 = x5863 + z5864 = x5864 + z5865 = x5865 + z5866 = x5866 + z5867 = x5867 + z5868 = x5868 + z5869 = x5869 + z5870 = x5870 + z5871 = x5871 + z5872 = x5872 + z5873 = x5873 + z5874 = x5874 + z5875 = x5875 + z5876 = x5876 + z5877 = x5877 + z5878 = x5878 + z5879 = x5879 + z5880 = x5880 + z5881 = x5881 + z5882 = x5882 + z5883 = x5883 + z5884 = x5884 + z5885 = x5885 + z5886 = x5886 + z5887 = x5887 + z5888 = x5888 + z5889 = x5889 + z5890 = x5890 + z5891 = x5891 + z5892 = x5892 + z5893 = x5893 + z5894 = x5894 + z5895 = x5895 + z5896 = x5896 + z5897 = x5897 + z5898 = x5898 + z5899 = x5899 + z5900 = x5900 + z5901 = x5901 + z5902 = x5902 + z5903 = x5903 + z5904 = x5904 + z5905 = x5905 + z5906 = x5906 + z5907 = x5907 + z5908 = x5908 + z5909 = x5909 + z5910 = x5910 + z5911 = x5911 + z5912 = x5912 + z5913 = x5913 + z5914 = x5914 + z5915 = x5915 + z5916 = x5916 + z5917 = x5917 + z5918 = x5918 + z5919 = x5919 + z5920 = x5920 + z5921 = x5921 + z5922 = x5922 + z5923 = x5923 + z5924 = x5924 + z5925 = x5925 + z5926 = x5926 + z5927 = x5927 + z5928 = x5928 + z5929 = x5929 + z5930 = x5930 + z5931 = x5931 + z5932 = x5932 + z5933 = x5933 + z5934 = x5934 + z5935 = x5935 + z5936 = x5936 + z5937 = x5937 + z5938 = x5938 + z5939 = x5939 + z5940 = x5940 + z5941 = x5941 + z5942 = x5942 + z5943 = x5943 + z5944 = x5944 + z5945 = x5945 + z5946 = x5946 + z5947 = x5947 + z5948 = x5948 + z5949 = x5949 + z5950 = x5950 + z5951 = x5951 + z5952 = x5952 + z5953 = x5953 + z5954 = x5954 + z5955 = x5955 + z5956 = x5956 + z5957 = x5957 + z5958 = x5958 + z5959 = x5959 + z5960 = x5960 + z5961 = x5961 + z5962 = x5962 + z5963 = x5963 + z5964 = x5964 + z5965 = x5965 + z5966 = x5966 + z5967 = x5967 + z5968 = x5968 + z5969 = x5969 + z5970 = x5970 + z5971 = x5971 + z5972 = x5972 + z5973 = x5973 + z5974 = x5974 + z5975 = x5975 + z5976 = x5976 + z5977 = x5977 + z5978 = x5978 + z5979 = x5979 + z5980 = x5980 + z5981 = x5981 + z5982 = x5982 + z5983 = x5983 + z5984 = x5984 + z5985 = x5985 + z5986 = x5986 + z5987 = x5987 + z5988 = x5988 + z5989 = x5989 + z5990 = x5990 + z5991 = x5991 + z5992 = x5992 + z5993 = x5993 + z5994 = x5994 + z5995 = x5995 + z5996 = x5996 + z5997 = x5997 + z5998 = x5998 + z5999 = x5999 + z6000 = x6000 + z6001 = x6001 + z6002 = x6002 + z6003 = x6003 + z6004 = x6004 + z6005 = x6005 + z6006 = x6006 + z6007 = x6007 + z6008 = x6008 + z6009 = x6009 + z6010 = x6010 + z6011 = x6011 + z6012 = x6012 + z6013 = x6013 + z6014 = x6014 + z6015 = x6015 + z6016 = x6016 + z6017 = x6017 + z6018 = x6018 + z6019 = x6019 + z6020 = x6020 + z6021 = x6021 + z6022 = x6022 + z6023 = x6023 + z6024 = x6024 + z6025 = x6025 + z6026 = x6026 + z6027 = x6027 + z6028 = x6028 + z6029 = x6029 + z6030 = x6030 + z6031 = x6031 + z6032 = x6032 + z6033 = x6033 + z6034 = x6034 + z6035 = x6035 + z6036 = x6036 + z6037 = x6037 + z6038 = x6038 + z6039 = x6039 + z6040 = x6040 + z6041 = x6041 + z6042 = x6042 + z6043 = x6043 + z6044 = x6044 + z6045 = x6045 + z6046 = x6046 + z6047 = x6047 + z6048 = x6048 + z6049 = x6049 + z6050 = x6050 + z6051 = x6051 + z6052 = x6052 + z6053 = x6053 + z6054 = x6054 + z6055 = x6055 + z6056 = x6056 + z6057 = x6057 + z6058 = x6058 + z6059 = x6059 + z6060 = x6060 + z6061 = x6061 + z6062 = x6062 + z6063 = x6063 + z6064 = x6064 + z6065 = x6065 + z6066 = x6066 + z6067 = x6067 + z6068 = x6068 + z6069 = x6069 + z6070 = x6070 + z6071 = x6071 + z6072 = x6072 + z6073 = x6073 + z6074 = x6074 + z6075 = x6075 + z6076 = x6076 + z6077 = x6077 + z6078 = x6078 + z6079 = x6079 + z6080 = x6080 + z6081 = x6081 + z6082 = x6082 + z6083 = x6083 + z6084 = x6084 + z6085 = x6085 + z6086 = x6086 + z6087 = x6087 + z6088 = x6088 + z6089 = x6089 + z6090 = x6090 + z6091 = x6091 + z6092 = x6092 + z6093 = x6093 + z6094 = x6094 + z6095 = x6095 + z6096 = x6096 + z6097 = x6097 + z6098 = x6098 + z6099 = x6099 + z6100 = x6100 + z6101 = x6101 + z6102 = x6102 + z6103 = x6103 + z6104 = x6104 + z6105 = x6105 + z6106 = x6106 + z6107 = x6107 + z6108 = x6108 + z6109 = x6109 + z6110 = x6110 + z6111 = x6111 + z6112 = x6112 + z6113 = x6113 + z6114 = x6114 + z6115 = x6115 + z6116 = x6116 + z6117 = x6117 + z6118 = x6118 + z6119 = x6119 + z6120 = x6120 + z6121 = x6121 + z6122 = x6122 + z6123 = x6123 + z6124 = x6124 + z6125 = x6125 + z6126 = x6126 + z6127 = x6127 + z6128 = x6128 + z6129 = x6129 + z6130 = x6130 + z6131 = x6131 + z6132 = x6132 + z6133 = x6133 + z6134 = x6134 + z6135 = x6135 + z6136 = x6136 + z6137 = x6137 + z6138 = x6138 + z6139 = x6139 + z6140 = x6140 + z6141 = x6141 + z6142 = x6142 + z6143 = x6143 + z6144 = x6144 + z6145 = x6145 + z6146 = x6146 + z6147 = x6147 + z6148 = x6148 + z6149 = x6149 + z6150 = x6150 + z6151 = x6151 + z6152 = x6152 + z6153 = x6153 + z6154 = x6154 + z6155 = x6155 + z6156 = x6156 + z6157 = x6157 + z6158 = x6158 + z6159 = x6159 + z6160 = x6160 + z6161 = x6161 + z6162 = x6162 + z6163 = x6163 + z6164 = x6164 + z6165 = x6165 + z6166 = x6166 + z6167 = x6167 + z6168 = x6168 + z6169 = x6169 + z6170 = x6170 + z6171 = x6171 + z6172 = x6172 + z6173 = x6173 + z6174 = x6174 + z6175 = x6175 + z6176 = x6176 + z6177 = x6177 + z6178 = x6178 + z6179 = x6179 + z6180 = x6180 + z6181 = x6181 + z6182 = x6182 + z6183 = x6183 + z6184 = x6184 + z6185 = x6185 + z6186 = x6186 + z6187 = x6187 + z6188 = x6188 + z6189 = x6189 + z6190 = x6190 + z6191 = x6191 + z6192 = x6192 + z6193 = x6193 + z6194 = x6194 + z6195 = x6195 + z6196 = x6196 + z6197 = x6197 + z6198 = x6198 + z6199 = x6199 + z6200 = x6200 + z6201 = x6201 + z6202 = x6202 + z6203 = x6203 + z6204 = x6204 + z6205 = x6205 + z6206 = x6206 + z6207 = x6207 + z6208 = x6208 + z6209 = x6209 + z6210 = x6210 + z6211 = x6211 + z6212 = x6212 + z6213 = x6213 + z6214 = x6214 + z6215 = x6215 + z6216 = x6216 + z6217 = x6217 + z6218 = x6218 + z6219 = x6219 + z6220 = x6220 + z6221 = x6221 + z6222 = x6222 + z6223 = x6223 + z6224 = x6224 + z6225 = x6225 + z6226 = x6226 + z6227 = x6227 + z6228 = x6228 + z6229 = x6229 + z6230 = x6230 + z6231 = x6231 + z6232 = x6232 + z6233 = x6233 + z6234 = x6234 + z6235 = x6235 + z6236 = x6236 + z6237 = x6237 + z6238 = x6238 + z6239 = x6239 + z6240 = x6240 + z6241 = x6241 + z6242 = x6242 + z6243 = x6243 + z6244 = x6244 + z6245 = x6245 + z6246 = x6246 + z6247 = x6247 + z6248 = x6248 + z6249 = x6249 + z6250 = x6250 + z6251 = x6251 + z6252 = x6252 + z6253 = x6253 + z6254 = x6254 + z6255 = x6255 + z6256 = x6256 + z6257 = x6257 + z6258 = x6258 + z6259 = x6259 + z6260 = x6260 + z6261 = x6261 + z6262 = x6262 + z6263 = x6263 + z6264 = x6264 + z6265 = x6265 + z6266 = x6266 + z6267 = x6267 + z6268 = x6268 + z6269 = x6269 + z6270 = x6270 + z6271 = x6271 + z6272 = x6272 + z6273 = x6273 + z6274 = x6274 + z6275 = x6275 + z6276 = x6276 + z6277 = x6277 + z6278 = x6278 + z6279 = x6279 + z6280 = x6280 + z6281 = x6281 + z6282 = x6282 + z6283 = x6283 + z6284 = x6284 + z6285 = x6285 + z6286 = x6286 + z6287 = x6287 + z6288 = x6288 + z6289 = x6289 + z6290 = x6290 + z6291 = x6291 + z6292 = x6292 + z6293 = x6293 + z6294 = x6294 + z6295 = x6295 + z6296 = x6296 + z6297 = x6297 + z6298 = x6298 + z6299 = x6299 + z6300 = x6300 + z6301 = x6301 + z6302 = x6302 + z6303 = x6303 + z6304 = x6304 + z6305 = x6305 + z6306 = x6306 + z6307 = x6307 + z6308 = x6308 + z6309 = x6309 + z6310 = x6310 + z6311 = x6311 + z6312 = x6312 + z6313 = x6313 + z6314 = x6314 + z6315 = x6315 + z6316 = x6316 + z6317 = x6317 + z6318 = x6318 + z6319 = x6319 + z6320 = x6320 + z6321 = x6321 + z6322 = x6322 + z6323 = x6323 + z6324 = x6324 + z6325 = x6325 + z6326 = x6326 + z6327 = x6327 + z6328 = x6328 + z6329 = x6329 + z6330 = x6330 + z6331 = x6331 + z6332 = x6332 + z6333 = x6333 + z6334 = x6334 + z6335 = x6335 + z6336 = x6336 + z6337 = x6337 + z6338 = x6338 + z6339 = x6339 + z6340 = x6340 + z6341 = x6341 + z6342 = x6342 + z6343 = x6343 + z6344 = x6344 + z6345 = x6345 + z6346 = x6346 + z6347 = x6347 + z6348 = x6348 + z6349 = x6349 + z6350 = x6350 + z6351 = x6351 + z6352 = x6352 + z6353 = x6353 + z6354 = x6354 + z6355 = x6355 + z6356 = x6356 + z6357 = x6357 + z6358 = x6358 + z6359 = x6359 + z6360 = x6360 + z6361 = x6361 + z6362 = x6362 + z6363 = x6363 + z6364 = x6364 + z6365 = x6365 + z6366 = x6366 + z6367 = x6367 + z6368 = x6368 + z6369 = x6369 + z6370 = x6370 + z6371 = x6371 + z6372 = x6372 + z6373 = x6373 + z6374 = x6374 + z6375 = x6375 + z6376 = x6376 + z6377 = x6377 + z6378 = x6378 + z6379 = x6379 + z6380 = x6380 + z6381 = x6381 + z6382 = x6382 + z6383 = x6383 + z6384 = x6384 + z6385 = x6385 + z6386 = x6386 + z6387 = x6387 + z6388 = x6388 + z6389 = x6389 + z6390 = x6390 + z6391 = x6391 + z6392 = x6392 + z6393 = x6393 + z6394 = x6394 + z6395 = x6395 + z6396 = x6396 + z6397 = x6397 + z6398 = x6398 + z6399 = x6399 + z6400 = x6400 + z6401 = x6401 + z6402 = x6402 + z6403 = x6403 + z6404 = x6404 + z6405 = x6405 + z6406 = x6406 + z6407 = x6407 + z6408 = x6408 + z6409 = x6409 + z6410 = x6410 + z6411 = x6411 + z6412 = x6412 + z6413 = x6413 + z6414 = x6414 + z6415 = x6415 + z6416 = x6416 + z6417 = x6417 + z6418 = x6418 + z6419 = x6419 + z6420 = x6420 + z6421 = x6421 + z6422 = x6422 + z6423 = x6423 + z6424 = x6424 + z6425 = x6425 + z6426 = x6426 + z6427 = x6427 + z6428 = x6428 + z6429 = x6429 + z6430 = x6430 + z6431 = x6431 + z6432 = x6432 + z6433 = x6433 + z6434 = x6434 + z6435 = x6435 + z6436 = x6436 + z6437 = x6437 + z6438 = x6438 + z6439 = x6439 + z6440 = x6440 + z6441 = x6441 + z6442 = x6442 + z6443 = x6443 + z6444 = x6444 + z6445 = x6445 + z6446 = x6446 + z6447 = x6447 + z6448 = x6448 + z6449 = x6449 + z6450 = x6450 + z6451 = x6451 + z6452 = x6452 + z6453 = x6453 + z6454 = x6454 + z6455 = x6455 + z6456 = x6456 + z6457 = x6457 + z6458 = x6458 + z6459 = x6459 + z6460 = x6460 + z6461 = x6461 + z6462 = x6462 + z6463 = x6463 + z6464 = x6464 + z6465 = x6465 + z6466 = x6466 + z6467 = x6467 + z6468 = x6468 + z6469 = x6469 + z6470 = x6470 + z6471 = x6471 + z6472 = x6472 + z6473 = x6473 + z6474 = x6474 + z6475 = x6475 + z6476 = x6476 + z6477 = x6477 + z6478 = x6478 + z6479 = x6479 + z6480 = x6480 + z6481 = x6481 + z6482 = x6482 + z6483 = x6483 + z6484 = x6484 + z6485 = x6485 + z6486 = x6486 + z6487 = x6487 + z6488 = x6488 + z6489 = x6489 + z6490 = x6490 + z6491 = x6491 + z6492 = x6492 + z6493 = x6493 + z6494 = x6494 + z6495 = x6495 + z6496 = x6496 + z6497 = x6497 + z6498 = x6498 + z6499 = x6499 + z6500 = x6500 + z6501 = x6501 + z6502 = x6502 + z6503 = x6503 + z6504 = x6504 + z6505 = x6505 + z6506 = x6506 + z6507 = x6507 + z6508 = x6508 + z6509 = x6509 + z6510 = x6510 + z6511 = x6511 + z6512 = x6512 + z6513 = x6513 + z6514 = x6514 + z6515 = x6515 + z6516 = x6516 + z6517 = x6517 + z6518 = x6518 + z6519 = x6519 + z6520 = x6520 + z6521 = x6521 + z6522 = x6522 + z6523 = x6523 + z6524 = x6524 + z6525 = x6525 + z6526 = x6526 + z6527 = x6527 + z6528 = x6528 + z6529 = x6529 + z6530 = x6530 + z6531 = x6531 + z6532 = x6532 + z6533 = x6533 + z6534 = x6534 + z6535 = x6535 + z6536 = x6536 + z6537 = x6537 + z6538 = x6538 + z6539 = x6539 + z6540 = x6540 + z6541 = x6541 + z6542 = x6542 + z6543 = x6543 + z6544 = x6544 + z6545 = x6545 + z6546 = x6546 + z6547 = x6547 + z6548 = x6548 + z6549 = x6549 + z6550 = x6550 + z6551 = x6551 + z6552 = x6552 + z6553 = x6553 + z6554 = x6554 + z6555 = x6555 + z6556 = x6556 + z6557 = x6557 + z6558 = x6558 + z6559 = x6559 + z6560 = x6560 + z6561 = x6561 + z6562 = x6562 + z6563 = x6563 + z6564 = x6564 + z6565 = x6565 + z6566 = x6566 + z6567 = x6567 + z6568 = x6568 + z6569 = x6569 + z6570 = x6570 + z6571 = x6571 + z6572 = x6572 + z6573 = x6573 + z6574 = x6574 + z6575 = x6575 + z6576 = x6576 + z6577 = x6577 + z6578 = x6578 + z6579 = x6579 + z6580 = x6580 + z6581 = x6581 + z6582 = x6582 + z6583 = x6583 + z6584 = x6584 + z6585 = x6585 + z6586 = x6586 + z6587 = x6587 + z6588 = x6588 + z6589 = x6589 + z6590 = x6590 + z6591 = x6591 + z6592 = x6592 + z6593 = x6593 + z6594 = x6594 + z6595 = x6595 + z6596 = x6596 + z6597 = x6597 + z6598 = x6598 + z6599 = x6599 + z6600 = x6600 + z6601 = x6601 + z6602 = x6602 + z6603 = x6603 + z6604 = x6604 + z6605 = x6605 + z6606 = x6606 + z6607 = x6607 + z6608 = x6608 + z6609 = x6609 + z6610 = x6610 + z6611 = x6611 + z6612 = x6612 + z6613 = x6613 + z6614 = x6614 + z6615 = x6615 + z6616 = x6616 + z6617 = x6617 + z6618 = x6618 + z6619 = x6619 + z6620 = x6620 + z6621 = x6621 + z6622 = x6622 + z6623 = x6623 + z6624 = x6624 + z6625 = x6625 + z6626 = x6626 + z6627 = x6627 + z6628 = x6628 + z6629 = x6629 + z6630 = x6630 + z6631 = x6631 + z6632 = x6632 + z6633 = x6633 + z6634 = x6634 + z6635 = x6635 + z6636 = x6636 + z6637 = x6637 + z6638 = x6638 + z6639 = x6639 + z6640 = x6640 + z6641 = x6641 + z6642 = x6642 + z6643 = x6643 + z6644 = x6644 + z6645 = x6645 + z6646 = x6646 + z6647 = x6647 + z6648 = x6648 + z6649 = x6649 + z6650 = x6650 + z6651 = x6651 + z6652 = x6652 + z6653 = x6653 + z6654 = x6654 + z6655 = x6655 + z6656 = x6656 + z6657 = x6657 + z6658 = x6658 + z6659 = x6659 + z6660 = x6660 + z6661 = x6661 + z6662 = x6662 + z6663 = x6663 + z6664 = x6664 + z6665 = x6665 + z6666 = x6666 + z6667 = x6667 + z6668 = x6668 + z6669 = x6669 + z6670 = x6670 + z6671 = x6671 + z6672 = x6672 + z6673 = x6673 + z6674 = x6674 + z6675 = x6675 + z6676 = x6676 + z6677 = x6677 + z6678 = x6678 + z6679 = x6679 + z6680 = x6680 + z6681 = x6681 + z6682 = x6682 + z6683 = x6683 + z6684 = x6684 + z6685 = x6685 + z6686 = x6686 + z6687 = x6687 + z6688 = x6688 + z6689 = x6689 + z6690 = x6690 + z6691 = x6691 + z6692 = x6692 + z6693 = x6693 + z6694 = x6694 + z6695 = x6695 + z6696 = x6696 + z6697 = x6697 + z6698 = x6698 + z6699 = x6699 + z6700 = x6700 + z6701 = x6701 + z6702 = x6702 + z6703 = x6703 + z6704 = x6704 + z6705 = x6705 + z6706 = x6706 + z6707 = x6707 + z6708 = x6708 + z6709 = x6709 + z6710 = x6710 + z6711 = x6711 + z6712 = x6712 + z6713 = x6713 + z6714 = x6714 + z6715 = x6715 + z6716 = x6716 + z6717 = x6717 + z6718 = x6718 + z6719 = x6719 + z6720 = x6720 + z6721 = x6721 + z6722 = x6722 + z6723 = x6723 + z6724 = x6724 + z6725 = x6725 + z6726 = x6726 + z6727 = x6727 + z6728 = x6728 + z6729 = x6729 + z6730 = x6730 + z6731 = x6731 + z6732 = x6732 + z6733 = x6733 + z6734 = x6734 + z6735 = x6735 + z6736 = x6736 + z6737 = x6737 + z6738 = x6738 + z6739 = x6739 + z6740 = x6740 + z6741 = x6741 + z6742 = x6742 + z6743 = x6743 + z6744 = x6744 + z6745 = x6745 + z6746 = x6746 + z6747 = x6747 + z6748 = x6748 + z6749 = x6749 + z6750 = x6750 + z6751 = x6751 + z6752 = x6752 + z6753 = x6753 + z6754 = x6754 + z6755 = x6755 + z6756 = x6756 + z6757 = x6757 + z6758 = x6758 + z6759 = x6759 + z6760 = x6760 + z6761 = x6761 + z6762 = x6762 + z6763 = x6763 + z6764 = x6764 + z6765 = x6765 + z6766 = x6766 + z6767 = x6767 + z6768 = x6768 + z6769 = x6769 + z6770 = x6770 + z6771 = x6771 + z6772 = x6772 + z6773 = x6773 + z6774 = x6774 + z6775 = x6775 + z6776 = x6776 + z6777 = x6777 + z6778 = x6778 + z6779 = x6779 + z6780 = x6780 + z6781 = x6781 + z6782 = x6782 + z6783 = x6783 + z6784 = x6784 + z6785 = x6785 + z6786 = x6786 + z6787 = x6787 + z6788 = x6788 + z6789 = x6789 + z6790 = x6790 + z6791 = x6791 + z6792 = x6792 + z6793 = x6793 + z6794 = x6794 + z6795 = x6795 + z6796 = x6796 + z6797 = x6797 + z6798 = x6798 + z6799 = x6799 + z6800 = x6800 + z6801 = x6801 + z6802 = x6802 + z6803 = x6803 + z6804 = x6804 + z6805 = x6805 + z6806 = x6806 + z6807 = x6807 + z6808 = x6808 + z6809 = x6809 + z6810 = x6810 + z6811 = x6811 + z6812 = x6812 + z6813 = x6813 + z6814 = x6814 + z6815 = x6815 + z6816 = x6816 + z6817 = x6817 + z6818 = x6818 + z6819 = x6819 + z6820 = x6820 + z6821 = x6821 + z6822 = x6822 + z6823 = x6823 + z6824 = x6824 + z6825 = x6825 + z6826 = x6826 + z6827 = x6827 + z6828 = x6828 + z6829 = x6829 + z6830 = x6830 + z6831 = x6831 + z6832 = x6832 + z6833 = x6833 + z6834 = x6834 + z6835 = x6835 + z6836 = x6836 + z6837 = x6837 + z6838 = x6838 + z6839 = x6839 + z6840 = x6840 + z6841 = x6841 + z6842 = x6842 + z6843 = x6843 + z6844 = x6844 + z6845 = x6845 + z6846 = x6846 + z6847 = x6847 + z6848 = x6848 + z6849 = x6849 + z6850 = x6850 + z6851 = x6851 + z6852 = x6852 + z6853 = x6853 + z6854 = x6854 + z6855 = x6855 + z6856 = x6856 + z6857 = x6857 + z6858 = x6858 + z6859 = x6859 + z6860 = x6860 + z6861 = x6861 + z6862 = x6862 + z6863 = x6863 + z6864 = x6864 + z6865 = x6865 + z6866 = x6866 + z6867 = x6867 + z6868 = x6868 + z6869 = x6869 + z6870 = x6870 + z6871 = x6871 + z6872 = x6872 + z6873 = x6873 + z6874 = x6874 + z6875 = x6875 + z6876 = x6876 + z6877 = x6877 + z6878 = x6878 + z6879 = x6879 + z6880 = x6880 + z6881 = x6881 + z6882 = x6882 + z6883 = x6883 + z6884 = x6884 + z6885 = x6885 + z6886 = x6886 + z6887 = x6887 + z6888 = x6888 + z6889 = x6889 + z6890 = x6890 + z6891 = x6891 + z6892 = x6892 + z6893 = x6893 + z6894 = x6894 + z6895 = x6895 + z6896 = x6896 + z6897 = x6897 + z6898 = x6898 + z6899 = x6899 + z6900 = x6900 + z6901 = x6901 + z6902 = x6902 + z6903 = x6903 + z6904 = x6904 + z6905 = x6905 + z6906 = x6906 + z6907 = x6907 + z6908 = x6908 + z6909 = x6909 + z6910 = x6910 + z6911 = x6911 + z6912 = x6912 + z6913 = x6913 + z6914 = x6914 + z6915 = x6915 + z6916 = x6916 + z6917 = x6917 + z6918 = x6918 + z6919 = x6919 + z6920 = x6920 + z6921 = x6921 + z6922 = x6922 + z6923 = x6923 + z6924 = x6924 + z6925 = x6925 + z6926 = x6926 + z6927 = x6927 + z6928 = x6928 + z6929 = x6929 + z6930 = x6930 + z6931 = x6931 + z6932 = x6932 + z6933 = x6933 + z6934 = x6934 + z6935 = x6935 + z6936 = x6936 + z6937 = x6937 + z6938 = x6938 + z6939 = x6939 + z6940 = x6940 + z6941 = x6941 + z6942 = x6942 + z6943 = x6943 + z6944 = x6944 + z6945 = x6945 + z6946 = x6946 + z6947 = x6947 + z6948 = x6948 + z6949 = x6949 + z6950 = x6950 + z6951 = x6951 + z6952 = x6952 + z6953 = x6953 + z6954 = x6954 + z6955 = x6955 + z6956 = x6956 + z6957 = x6957 + z6958 = x6958 + z6959 = x6959 + z6960 = x6960 + z6961 = x6961 + z6962 = x6962 + z6963 = x6963 + z6964 = x6964 + z6965 = x6965 + z6966 = x6966 + z6967 = x6967 + z6968 = x6968 + z6969 = x6969 + z6970 = x6970 + z6971 = x6971 + z6972 = x6972 + z6973 = x6973 + z6974 = x6974 + z6975 = x6975 + z6976 = x6976 + z6977 = x6977 + z6978 = x6978 + z6979 = x6979 + z6980 = x6980 + z6981 = x6981 + z6982 = x6982 + z6983 = x6983 + z6984 = x6984 + z6985 = x6985 + z6986 = x6986 + z6987 = x6987 + z6988 = x6988 + z6989 = x6989 + z6990 = x6990 + z6991 = x6991 + z6992 = x6992 + z6993 = x6993 + z6994 = x6994 + z6995 = x6995 + z6996 = x6996 + z6997 = x6997 + z6998 = x6998 + z6999 = x6999 + z7000 = x7000 + z7001 = x7001 + z7002 = x7002 + z7003 = x7003 + z7004 = x7004 + z7005 = x7005 + z7006 = x7006 + z7007 = x7007 + z7008 = x7008 + z7009 = x7009 + z7010 = x7010 + z7011 = x7011 + z7012 = x7012 + z7013 = x7013 + z7014 = x7014 + z7015 = x7015 + z7016 = x7016 + z7017 = x7017 + z7018 = x7018 + z7019 = x7019 + z7020 = x7020 + z7021 = x7021 + z7022 = x7022 + z7023 = x7023 + z7024 = x7024 + z7025 = x7025 + z7026 = x7026 + z7027 = x7027 + z7028 = x7028 + z7029 = x7029 + z7030 = x7030 + z7031 = x7031 + z7032 = x7032 + z7033 = x7033 + z7034 = x7034 + z7035 = x7035 + z7036 = x7036 + z7037 = x7037 + z7038 = x7038 + z7039 = x7039 + z7040 = x7040 + z7041 = x7041 + z7042 = x7042 + z7043 = x7043 + z7044 = x7044 + z7045 = x7045 + z7046 = x7046 + z7047 = x7047 + z7048 = x7048 + z7049 = x7049 + z7050 = x7050 + z7051 = x7051 + z7052 = x7052 + z7053 = x7053 + z7054 = x7054 + z7055 = x7055 + z7056 = x7056 + z7057 = x7057 + z7058 = x7058 + z7059 = x7059 + z7060 = x7060 + z7061 = x7061 + z7062 = x7062 + z7063 = x7063 + z7064 = x7064 + z7065 = x7065 + z7066 = x7066 + z7067 = x7067 + z7068 = x7068 + z7069 = x7069 + z7070 = x7070 + z7071 = x7071 + z7072 = x7072 + z7073 = x7073 + z7074 = x7074 + z7075 = x7075 + z7076 = x7076 + z7077 = x7077 + z7078 = x7078 + z7079 = x7079 + z7080 = x7080 + z7081 = x7081 + z7082 = x7082 + z7083 = x7083 + z7084 = x7084 + z7085 = x7085 + z7086 = x7086 + z7087 = x7087 + z7088 = x7088 + z7089 = x7089 + z7090 = x7090 + z7091 = x7091 + z7092 = x7092 + z7093 = x7093 + z7094 = x7094 + z7095 = x7095 + z7096 = x7096 + z7097 = x7097 + z7098 = x7098 + z7099 = x7099 + z7100 = x7100 + z7101 = x7101 + z7102 = x7102 + z7103 = x7103 + z7104 = x7104 + z7105 = x7105 + z7106 = x7106 + z7107 = x7107 + z7108 = x7108 + z7109 = x7109 + z7110 = x7110 + z7111 = x7111 + z7112 = x7112 + z7113 = x7113 + z7114 = x7114 + z7115 = x7115 + z7116 = x7116 + z7117 = x7117 + z7118 = x7118 + z7119 = x7119 + z7120 = x7120 + z7121 = x7121 + z7122 = x7122 + z7123 = x7123 + z7124 = x7124 + z7125 = x7125 + z7126 = x7126 + z7127 = x7127 + z7128 = x7128 + z7129 = x7129 + z7130 = x7130 + z7131 = x7131 + z7132 = x7132 + z7133 = x7133 + z7134 = x7134 + z7135 = x7135 + z7136 = x7136 + z7137 = x7137 + z7138 = x7138 + z7139 = x7139 + z7140 = x7140 + z7141 = x7141 + z7142 = x7142 + z7143 = x7143 + z7144 = x7144 + z7145 = x7145 + z7146 = x7146 + z7147 = x7147 + z7148 = x7148 + z7149 = x7149 + z7150 = x7150 + z7151 = x7151 + z7152 = x7152 + z7153 = x7153 + z7154 = x7154 + z7155 = x7155 + z7156 = x7156 + z7157 = x7157 + z7158 = x7158 + z7159 = x7159 + z7160 = x7160 + z7161 = x7161 + z7162 = x7162 + z7163 = x7163 + z7164 = x7164 + z7165 = x7165 + z7166 = x7166 + z7167 = x7167 + z7168 = x7168 + z7169 = x7169 + z7170 = x7170 + z7171 = x7171 + z7172 = x7172 + z7173 = x7173 + z7174 = x7174 + z7175 = x7175 + z7176 = x7176 + z7177 = x7177 + z7178 = x7178 + z7179 = x7179 + z7180 = x7180 + z7181 = x7181 + z7182 = x7182 + z7183 = x7183 + z7184 = x7184 + z7185 = x7185 + z7186 = x7186 + z7187 = x7187 + z7188 = x7188 + z7189 = x7189 + z7190 = x7190 + z7191 = x7191 + z7192 = x7192 + z7193 = x7193 + z7194 = x7194 + z7195 = x7195 + z7196 = x7196 + z7197 = x7197 + z7198 = x7198 + z7199 = x7199 + z7200 = x7200 + z7201 = x7201 + z7202 = x7202 + z7203 = x7203 + z7204 = x7204 + z7205 = x7205 + z7206 = x7206 + z7207 = x7207 + z7208 = x7208 + z7209 = x7209 + z7210 = x7210 + z7211 = x7211 + z7212 = x7212 + z7213 = x7213 + z7214 = x7214 + z7215 = x7215 + z7216 = x7216 + z7217 = x7217 + z7218 = x7218 + z7219 = x7219 + z7220 = x7220 + z7221 = x7221 + z7222 = x7222 + z7223 = x7223 + z7224 = x7224 + z7225 = x7225 + z7226 = x7226 + z7227 = x7227 + z7228 = x7228 + z7229 = x7229 + z7230 = x7230 + z7231 = x7231 + z7232 = x7232 + z7233 = x7233 + z7234 = x7234 + z7235 = x7235 + z7236 = x7236 + z7237 = x7237 + z7238 = x7238 + z7239 = x7239 + z7240 = x7240 + z7241 = x7241 + z7242 = x7242 + z7243 = x7243 + z7244 = x7244 + z7245 = x7245 + z7246 = x7246 + z7247 = x7247 + z7248 = x7248 + z7249 = x7249 + z7250 = x7250 + z7251 = x7251 + z7252 = x7252 + z7253 = x7253 + z7254 = x7254 + z7255 = x7255 + z7256 = x7256 + z7257 = x7257 + z7258 = x7258 + z7259 = x7259 + z7260 = x7260 + z7261 = x7261 + z7262 = x7262 + z7263 = x7263 + z7264 = x7264 + z7265 = x7265 + z7266 = x7266 + z7267 = x7267 + z7268 = x7268 + z7269 = x7269 + z7270 = x7270 + z7271 = x7271 + z7272 = x7272 + z7273 = x7273 + z7274 = x7274 + z7275 = x7275 + z7276 = x7276 + z7277 = x7277 + z7278 = x7278 + z7279 = x7279 + z7280 = x7280 + z7281 = x7281 + z7282 = x7282 + z7283 = x7283 + z7284 = x7284 + z7285 = x7285 + z7286 = x7286 + z7287 = x7287 + z7288 = x7288 + z7289 = x7289 + z7290 = x7290 + z7291 = x7291 + z7292 = x7292 + z7293 = x7293 + z7294 = x7294 + z7295 = x7295 + z7296 = x7296 + z7297 = x7297 + z7298 = x7298 + z7299 = x7299 + z7300 = x7300 + z7301 = x7301 + z7302 = x7302 + z7303 = x7303 + z7304 = x7304 + z7305 = x7305 + z7306 = x7306 + z7307 = x7307 + z7308 = x7308 + z7309 = x7309 + z7310 = x7310 + z7311 = x7311 + z7312 = x7312 + z7313 = x7313 + z7314 = x7314 + z7315 = x7315 + z7316 = x7316 + z7317 = x7317 + z7318 = x7318 + z7319 = x7319 + z7320 = x7320 + z7321 = x7321 + z7322 = x7322 + z7323 = x7323 + z7324 = x7324 + z7325 = x7325 + z7326 = x7326 + z7327 = x7327 + z7328 = x7328 + z7329 = x7329 + z7330 = x7330 + z7331 = x7331 + z7332 = x7332 + z7333 = x7333 + z7334 = x7334 + z7335 = x7335 + z7336 = x7336 + z7337 = x7337 + z7338 = x7338 + z7339 = x7339 + z7340 = x7340 + z7341 = x7341 + z7342 = x7342 + z7343 = x7343 + z7344 = x7344 + z7345 = x7345 + z7346 = x7346 + z7347 = x7347 + z7348 = x7348 + z7349 = x7349 + z7350 = x7350 + z7351 = x7351 + z7352 = x7352 + z7353 = x7353 + z7354 = x7354 + z7355 = x7355 + z7356 = x7356 + z7357 = x7357 + z7358 = x7358 + z7359 = x7359 + z7360 = x7360 + z7361 = x7361 + z7362 = x7362 + z7363 = x7363 + z7364 = x7364 + z7365 = x7365 + z7366 = x7366 + z7367 = x7367 + z7368 = x7368 + z7369 = x7369 + z7370 = x7370 + z7371 = x7371 + z7372 = x7372 + z7373 = x7373 + z7374 = x7374 + z7375 = x7375 + z7376 = x7376 + z7377 = x7377 + z7378 = x7378 + z7379 = x7379 + z7380 = x7380 + z7381 = x7381 + z7382 = x7382 + z7383 = x7383 + z7384 = x7384 + z7385 = x7385 + z7386 = x7386 + z7387 = x7387 + z7388 = x7388 + z7389 = x7389 + z7390 = x7390 + z7391 = x7391 + z7392 = x7392 + z7393 = x7393 + z7394 = x7394 + z7395 = x7395 + z7396 = x7396 + z7397 = x7397 + z7398 = x7398 + z7399 = x7399 + z7400 = x7400 + z7401 = x7401 + z7402 = x7402 + z7403 = x7403 + z7404 = x7404 + z7405 = x7405 + z7406 = x7406 + z7407 = x7407 + z7408 = x7408 + z7409 = x7409 + z7410 = x7410 + z7411 = x7411 + z7412 = x7412 + z7413 = x7413 + z7414 = x7414 + z7415 = x7415 + z7416 = x7416 + z7417 = x7417 + z7418 = x7418 + z7419 = x7419 + z7420 = x7420 + z7421 = x7421 + z7422 = x7422 + z7423 = x7423 + z7424 = x7424 + z7425 = x7425 + z7426 = x7426 + z7427 = x7427 + z7428 = x7428 + z7429 = x7429 + z7430 = x7430 + z7431 = x7431 + z7432 = x7432 + z7433 = x7433 + z7434 = x7434 + z7435 = x7435 + z7436 = x7436 + z7437 = x7437 + z7438 = x7438 + z7439 = x7439 + z7440 = x7440 + z7441 = x7441 + z7442 = x7442 + z7443 = x7443 + z7444 = x7444 + z7445 = x7445 + z7446 = x7446 + z7447 = x7447 + z7448 = x7448 + z7449 = x7449 + z7450 = x7450 + z7451 = x7451 + z7452 = x7452 + z7453 = x7453 + z7454 = x7454 + z7455 = x7455 + z7456 = x7456 + z7457 = x7457 + z7458 = x7458 + z7459 = x7459 + z7460 = x7460 + z7461 = x7461 + z7462 = x7462 + z7463 = x7463 + z7464 = x7464 + z7465 = x7465 + z7466 = x7466 + z7467 = x7467 + z7468 = x7468 + z7469 = x7469 + z7470 = x7470 + z7471 = x7471 + z7472 = x7472 + z7473 = x7473 + z7474 = x7474 + z7475 = x7475 + z7476 = x7476 + z7477 = x7477 + z7478 = x7478 + z7479 = x7479 + z7480 = x7480 + z7481 = x7481 + z7482 = x7482 + z7483 = x7483 + z7484 = x7484 + z7485 = x7485 + z7486 = x7486 + z7487 = x7487 + z7488 = x7488 + z7489 = x7489 + z7490 = x7490 + z7491 = x7491 + z7492 = x7492 + z7493 = x7493 + z7494 = x7494 + z7495 = x7495 + z7496 = x7496 + z7497 = x7497 + z7498 = x7498 + z7499 = x7499 + z7500 = x7500 + z7501 = x7501 + z7502 = x7502 + z7503 = x7503 + z7504 = x7504 + z7505 = x7505 + z7506 = x7506 + z7507 = x7507 + z7508 = x7508 + z7509 = x7509 + z7510 = x7510 + z7511 = x7511 + z7512 = x7512 + z7513 = x7513 + z7514 = x7514 + z7515 = x7515 + z7516 = x7516 + z7517 = x7517 + z7518 = x7518 + z7519 = x7519 + z7520 = x7520 + z7521 = x7521 + z7522 = x7522 + z7523 = x7523 + z7524 = x7524 + z7525 = x7525 + z7526 = x7526 + z7527 = x7527 + z7528 = x7528 + z7529 = x7529 + z7530 = x7530 + z7531 = x7531 + z7532 = x7532 + z7533 = x7533 + z7534 = x7534 + z7535 = x7535 + z7536 = x7536 + z7537 = x7537 + z7538 = x7538 + z7539 = x7539 + z7540 = x7540 + z7541 = x7541 + z7542 = x7542 + z7543 = x7543 + z7544 = x7544 + z7545 = x7545 + z7546 = x7546 + z7547 = x7547 + z7548 = x7548 + z7549 = x7549 + z7550 = x7550 + z7551 = x7551 + z7552 = x7552 + z7553 = x7553 + z7554 = x7554 + z7555 = x7555 + z7556 = x7556 + z7557 = x7557 + z7558 = x7558 + z7559 = x7559 + z7560 = x7560 + z7561 = x7561 + z7562 = x7562 + z7563 = x7563 + z7564 = x7564 + z7565 = x7565 + z7566 = x7566 + z7567 = x7567 + z7568 = x7568 + z7569 = x7569 + z7570 = x7570 + z7571 = x7571 + z7572 = x7572 + z7573 = x7573 + z7574 = x7574 + z7575 = x7575 + z7576 = x7576 + z7577 = x7577 + z7578 = x7578 + z7579 = x7579 + z7580 = x7580 + z7581 = x7581 + z7582 = x7582 + z7583 = x7583 + z7584 = x7584 + z7585 = x7585 + z7586 = x7586 + z7587 = x7587 + z7588 = x7588 + z7589 = x7589 + z7590 = x7590 + z7591 = x7591 + z7592 = x7592 + z7593 = x7593 + z7594 = x7594 + z7595 = x7595 + z7596 = x7596 + z7597 = x7597 + z7598 = x7598 + z7599 = x7599 + z7600 = x7600 + z7601 = x7601 + z7602 = x7602 + z7603 = x7603 + z7604 = x7604 + z7605 = x7605 + z7606 = x7606 + z7607 = x7607 + z7608 = x7608 + z7609 = x7609 + z7610 = x7610 + z7611 = x7611 + z7612 = x7612 + z7613 = x7613 + z7614 = x7614 + z7615 = x7615 + z7616 = x7616 + z7617 = x7617 + z7618 = x7618 + z7619 = x7619 + z7620 = x7620 + z7621 = x7621 + z7622 = x7622 + z7623 = x7623 + z7624 = x7624 + z7625 = x7625 + z7626 = x7626 + z7627 = x7627 + z7628 = x7628 + z7629 = x7629 + z7630 = x7630 + z7631 = x7631 + z7632 = x7632 + z7633 = x7633 + z7634 = x7634 + z7635 = x7635 + z7636 = x7636 + z7637 = x7637 + z7638 = x7638 + z7639 = x7639 + z7640 = x7640 + z7641 = x7641 + z7642 = x7642 + z7643 = x7643 + z7644 = x7644 + z7645 = x7645 + z7646 = x7646 + z7647 = x7647 + z7648 = x7648 + z7649 = x7649 + z7650 = x7650 + z7651 = x7651 + z7652 = x7652 + z7653 = x7653 + z7654 = x7654 + z7655 = x7655 + z7656 = x7656 + z7657 = x7657 + z7658 = x7658 + z7659 = x7659 + z7660 = x7660 + z7661 = x7661 + z7662 = x7662 + z7663 = x7663 + z7664 = x7664 + z7665 = x7665 + z7666 = x7666 + z7667 = x7667 + z7668 = x7668 + z7669 = x7669 + z7670 = x7670 + z7671 = x7671 + z7672 = x7672 + z7673 = x7673 + z7674 = x7674 + z7675 = x7675 + z7676 = x7676 + z7677 = x7677 + z7678 = x7678 + z7679 = x7679 + z7680 = x7680 + z7681 = x7681 + z7682 = x7682 + z7683 = x7683 + z7684 = x7684 + z7685 = x7685 + z7686 = x7686 + z7687 = x7687 + z7688 = x7688 + z7689 = x7689 + z7690 = x7690 + z7691 = x7691 + z7692 = x7692 + z7693 = x7693 + z7694 = x7694 + z7695 = x7695 + z7696 = x7696 + z7697 = x7697 + z7698 = x7698 + z7699 = x7699 + z7700 = x7700 + z7701 = x7701 + z7702 = x7702 + z7703 = x7703 + z7704 = x7704 + z7705 = x7705 + z7706 = x7706 + z7707 = x7707 + z7708 = x7708 + z7709 = x7709 + z7710 = x7710 + z7711 = x7711 + z7712 = x7712 + z7713 = x7713 + z7714 = x7714 + z7715 = x7715 + z7716 = x7716 + z7717 = x7717 + z7718 = x7718 + z7719 = x7719 + z7720 = x7720 + z7721 = x7721 + z7722 = x7722 + z7723 = x7723 + z7724 = x7724 + z7725 = x7725 + z7726 = x7726 + z7727 = x7727 + z7728 = x7728 + z7729 = x7729 + z7730 = x7730 + z7731 = x7731 + z7732 = x7732 + z7733 = x7733 + z7734 = x7734 + z7735 = x7735 + z7736 = x7736 + z7737 = x7737 + z7738 = x7738 + z7739 = x7739 + z7740 = x7740 + z7741 = x7741 + z7742 = x7742 + z7743 = x7743 + z7744 = x7744 + z7745 = x7745 + z7746 = x7746 + z7747 = x7747 + z7748 = x7748 + z7749 = x7749 + z7750 = x7750 + z7751 = x7751 + z7752 = x7752 + z7753 = x7753 + z7754 = x7754 + z7755 = x7755 + z7756 = x7756 + z7757 = x7757 + z7758 = x7758 + z7759 = x7759 + z7760 = x7760 + z7761 = x7761 + z7762 = x7762 + z7763 = x7763 + z7764 = x7764 + z7765 = x7765 + z7766 = x7766 + z7767 = x7767 + z7768 = x7768 + z7769 = x7769 + z7770 = x7770 + z7771 = x7771 + z7772 = x7772 + z7773 = x7773 + z7774 = x7774 + z7775 = x7775 + z7776 = x7776 + z7777 = x7777 + z7778 = x7778 + z7779 = x7779 + z7780 = x7780 + z7781 = x7781 + z7782 = x7782 + z7783 = x7783 + z7784 = x7784 + z7785 = x7785 + z7786 = x7786 + z7787 = x7787 + z7788 = x7788 + z7789 = x7789 + z7790 = x7790 + z7791 = x7791 + z7792 = x7792 + z7793 = x7793 + z7794 = x7794 + z7795 = x7795 + z7796 = x7796 + z7797 = x7797 + z7798 = x7798 + z7799 = x7799 + z7800 = x7800 + z7801 = x7801 + z7802 = x7802 + z7803 = x7803 + z7804 = x7804 + z7805 = x7805 + z7806 = x7806 + z7807 = x7807 + z7808 = x7808 + z7809 = x7809 + z7810 = x7810 + z7811 = x7811 + z7812 = x7812 + z7813 = x7813 + z7814 = x7814 + z7815 = x7815 + z7816 = x7816 + z7817 = x7817 + z7818 = x7818 + z7819 = x7819 + z7820 = x7820 + z7821 = x7821 + z7822 = x7822 + z7823 = x7823 + z7824 = x7824 + z7825 = x7825 + z7826 = x7826 + z7827 = x7827 + z7828 = x7828 + z7829 = x7829 + z7830 = x7830 + z7831 = x7831 + z7832 = x7832 + z7833 = x7833 + z7834 = x7834 + z7835 = x7835 + z7836 = x7836 + z7837 = x7837 + z7838 = x7838 + z7839 = x7839 + z7840 = x7840 + z7841 = x7841 + z7842 = x7842 + z7843 = x7843 + z7844 = x7844 + z7845 = x7845 + z7846 = x7846 + z7847 = x7847 + z7848 = x7848 + z7849 = x7849 + z7850 = x7850 + z7851 = x7851 + z7852 = x7852 + z7853 = x7853 + z7854 = x7854 + z7855 = x7855 + z7856 = x7856 + z7857 = x7857 + z7858 = x7858 + z7859 = x7859 + z7860 = x7860 + z7861 = x7861 + z7862 = x7862 + z7863 = x7863 + z7864 = x7864 + z7865 = x7865 + z7866 = x7866 + z7867 = x7867 + z7868 = x7868 + z7869 = x7869 + z7870 = x7870 + z7871 = x7871 + z7872 = x7872 + z7873 = x7873 + z7874 = x7874 + z7875 = x7875 + z7876 = x7876 + z7877 = x7877 + z7878 = x7878 + z7879 = x7879 + z7880 = x7880 + z7881 = x7881 + z7882 = x7882 + z7883 = x7883 + z7884 = x7884 + z7885 = x7885 + z7886 = x7886 + z7887 = x7887 + z7888 = x7888 + z7889 = x7889 + z7890 = x7890 + z7891 = x7891 + z7892 = x7892 + z7893 = x7893 + z7894 = x7894 + z7895 = x7895 + z7896 = x7896 + z7897 = x7897 + z7898 = x7898 + z7899 = x7899 + z7900 = x7900 + z7901 = x7901 + z7902 = x7902 + z7903 = x7903 + z7904 = x7904 + z7905 = x7905 + z7906 = x7906 + z7907 = x7907 + z7908 = x7908 + z7909 = x7909 + z7910 = x7910 + z7911 = x7911 + z7912 = x7912 + z7913 = x7913 + z7914 = x7914 + z7915 = x7915 + z7916 = x7916 + z7917 = x7917 + z7918 = x7918 + z7919 = x7919 + z7920 = x7920 + z7921 = x7921 + z7922 = x7922 + z7923 = x7923 + z7924 = x7924 + z7925 = x7925 + z7926 = x7926 + z7927 = x7927 + z7928 = x7928 + z7929 = x7929 + z7930 = x7930 + z7931 = x7931 + z7932 = x7932 + z7933 = x7933 + z7934 = x7934 + z7935 = x7935 + z7936 = x7936 + z7937 = x7937 + z7938 = x7938 + z7939 = x7939 + z7940 = x7940 + z7941 = x7941 + z7942 = x7942 + z7943 = x7943 + z7944 = x7944 + z7945 = x7945 + z7946 = x7946 + z7947 = x7947 + z7948 = x7948 + z7949 = x7949 + z7950 = x7950 + z7951 = x7951 + z7952 = x7952 + z7953 = x7953 + z7954 = x7954 + z7955 = x7955 + z7956 = x7956 + z7957 = x7957 + z7958 = x7958 + z7959 = x7959 + z7960 = x7960 + z7961 = x7961 + z7962 = x7962 + z7963 = x7963 + z7964 = x7964 + z7965 = x7965 + z7966 = x7966 + z7967 = x7967 + z7968 = x7968 + z7969 = x7969 + z7970 = x7970 + z7971 = x7971 + z7972 = x7972 + z7973 = x7973 + z7974 = x7974 + z7975 = x7975 + z7976 = x7976 + z7977 = x7977 + z7978 = x7978 + z7979 = x7979 + z7980 = x7980 + z7981 = x7981 + z7982 = x7982 + z7983 = x7983 + z7984 = x7984 + z7985 = x7985 + z7986 = x7986 + z7987 = x7987 + z7988 = x7988 + z7989 = x7989 + z7990 = x7990 + z7991 = x7991 + z7992 = x7992 + z7993 = x7993 + z7994 = x7994 + z7995 = x7995 + z7996 = x7996 + z7997 = x7997 + z7998 = x7998 + z7999 = x7999 + z8000 = x8000 + z8001 = x8001 + z8002 = x8002 + z8003 = x8003 + z8004 = x8004 + z8005 = x8005 + z8006 = x8006 + z8007 = x8007 + z8008 = x8008 + z8009 = x8009 + z8010 = x8010 + z8011 = x8011 + z8012 = x8012 + z8013 = x8013 + z8014 = x8014 + z8015 = x8015 + z8016 = x8016 + z8017 = x8017 + z8018 = x8018 + z8019 = x8019 + z8020 = x8020 + z8021 = x8021 + z8022 = x8022 + z8023 = x8023 + z8024 = x8024 + z8025 = x8025 + z8026 = x8026 + z8027 = x8027 + z8028 = x8028 + z8029 = x8029 + z8030 = x8030 + z8031 = x8031 + z8032 = x8032 + z8033 = x8033 + z8034 = x8034 + z8035 = x8035 + z8036 = x8036 + z8037 = x8037 + z8038 = x8038 + z8039 = x8039 + z8040 = x8040 + z8041 = x8041 + z8042 = x8042 + z8043 = x8043 + z8044 = x8044 + z8045 = x8045 + z8046 = x8046 + z8047 = x8047 + z8048 = x8048 + z8049 = x8049 + z8050 = x8050 + z8051 = x8051 + z8052 = x8052 + z8053 = x8053 + z8054 = x8054 + z8055 = x8055 + z8056 = x8056 + z8057 = x8057 + z8058 = x8058 + z8059 = x8059 + z8060 = x8060 + z8061 = x8061 + z8062 = x8062 + z8063 = x8063 + z8064 = x8064 + z8065 = x8065 + z8066 = x8066 + z8067 = x8067 + z8068 = x8068 + z8069 = x8069 + z8070 = x8070 + z8071 = x8071 + z8072 = x8072 + z8073 = x8073 + z8074 = x8074 + z8075 = x8075 + z8076 = x8076 + z8077 = x8077 + z8078 = x8078 + z8079 = x8079 + z8080 = x8080 + z8081 = x8081 + z8082 = x8082 + z8083 = x8083 + z8084 = x8084 + z8085 = x8085 + z8086 = x8086 + z8087 = x8087 + z8088 = x8088 + z8089 = x8089 + z8090 = x8090 + z8091 = x8091 + z8092 = x8092 + z8093 = x8093 + z8094 = x8094 + z8095 = x8095 + z8096 = x8096 + z8097 = x8097 + z8098 = x8098 + z8099 = x8099 + z8100 = x8100 + z8101 = x8101 + z8102 = x8102 + z8103 = x8103 + z8104 = x8104 + z8105 = x8105 + z8106 = x8106 + z8107 = x8107 + z8108 = x8108 + z8109 = x8109 + z8110 = x8110 + z8111 = x8111 + z8112 = x8112 + z8113 = x8113 + z8114 = x8114 + z8115 = x8115 + z8116 = x8116 + z8117 = x8117 + z8118 = x8118 + z8119 = x8119 + z8120 = x8120 + z8121 = x8121 + z8122 = x8122 + z8123 = x8123 + z8124 = x8124 + z8125 = x8125 + z8126 = x8126 + z8127 = x8127 + z8128 = x8128 + z8129 = x8129 + z8130 = x8130 + z8131 = x8131 + z8132 = x8132 + z8133 = x8133 + z8134 = x8134 + z8135 = x8135 + z8136 = x8136 + z8137 = x8137 + z8138 = x8138 + z8139 = x8139 + z8140 = x8140 + z8141 = x8141 + z8142 = x8142 + z8143 = x8143 + z8144 = x8144 + z8145 = x8145 + z8146 = x8146 + z8147 = x8147 + z8148 = x8148 + z8149 = x8149 + z8150 = x8150 + z8151 = x8151 + z8152 = x8152 + z8153 = x8153 + z8154 = x8154 + z8155 = x8155 + z8156 = x8156 + z8157 = x8157 + z8158 = x8158 + z8159 = x8159 + z8160 = x8160 + z8161 = x8161 + z8162 = x8162 + z8163 = x8163 + z8164 = x8164 + z8165 = x8165 + z8166 = x8166 + z8167 = x8167 + z8168 = x8168 + z8169 = x8169 + z8170 = x8170 + z8171 = x8171 + z8172 = x8172 + z8173 = x8173 + z8174 = x8174 + z8175 = x8175 + z8176 = x8176 + z8177 = x8177 + z8178 = x8178 + z8179 = x8179 + z8180 = x8180 + z8181 = x8181 + z8182 = x8182 + z8183 = x8183 + z8184 = x8184 + z8185 = x8185 + z8186 = x8186 + z8187 = x8187 + z8188 = x8188 + z8189 = x8189 + z8190 = x8190 + z8191 = x8191 + z8192 = x8192 + z8193 = x8193 + z8194 = x8194 + z8195 = x8195 + z8196 = x8196 + z8197 = x8197 + z8198 = x8198 + z8199 = x8199 + z8200 = x8200 + z8201 = x8201 + z8202 = x8202 + z8203 = x8203 + z8204 = x8204 + z8205 = x8205 + z8206 = x8206 + z8207 = x8207 + z8208 = x8208 + z8209 = x8209 + z8210 = x8210 + z8211 = x8211 + z8212 = x8212 + z8213 = x8213 + z8214 = x8214 + z8215 = x8215 + z8216 = x8216 + z8217 = x8217 + z8218 = x8218 + z8219 = x8219 + z8220 = x8220 + z8221 = x8221 + z8222 = x8222 + z8223 = x8223 + z8224 = x8224 + z8225 = x8225 + z8226 = x8226 + z8227 = x8227 + z8228 = x8228 + z8229 = x8229 + z8230 = x8230 + z8231 = x8231 + z8232 = x8232 + z8233 = x8233 + z8234 = x8234 + z8235 = x8235 + z8236 = x8236 + z8237 = x8237 + z8238 = x8238 + z8239 = x8239 + z8240 = x8240 + z8241 = x8241 + z8242 = x8242 + z8243 = x8243 + z8244 = x8244 + z8245 = x8245 + z8246 = x8246 + z8247 = x8247 + z8248 = x8248 + z8249 = x8249 + z8250 = x8250 + z8251 = x8251 + z8252 = x8252 + z8253 = x8253 + z8254 = x8254 + z8255 = x8255 + z8256 = x8256 + z8257 = x8257 + z8258 = x8258 + z8259 = x8259 + z8260 = x8260 + z8261 = x8261 + z8262 = x8262 + z8263 = x8263 + z8264 = x8264 + z8265 = x8265 + z8266 = x8266 + z8267 = x8267 + z8268 = x8268 + z8269 = x8269 + z8270 = x8270 + z8271 = x8271 + z8272 = x8272 + z8273 = x8273 + z8274 = x8274 + z8275 = x8275 + z8276 = x8276 + z8277 = x8277 + z8278 = x8278 + z8279 = x8279 + z8280 = x8280 + z8281 = x8281 + z8282 = x8282 + z8283 = x8283 + z8284 = x8284 + z8285 = x8285 + z8286 = x8286 + z8287 = x8287 + z8288 = x8288 + z8289 = x8289 + z8290 = x8290 + z8291 = x8291 + z8292 = x8292 + z8293 = x8293 + z8294 = x8294 + z8295 = x8295 + z8296 = x8296 + z8297 = x8297 + z8298 = x8298 + z8299 = x8299 + z8300 = x8300 + z8301 = x8301 + z8302 = x8302 + z8303 = x8303 + z8304 = x8304 + z8305 = x8305 + z8306 = x8306 + z8307 = x8307 + z8308 = x8308 + z8309 = x8309 + z8310 = x8310 + z8311 = x8311 + z8312 = x8312 + z8313 = x8313 + z8314 = x8314 + z8315 = x8315 + z8316 = x8316 + z8317 = x8317 + z8318 = x8318 + z8319 = x8319 + z8320 = x8320 + z8321 = x8321 + z8322 = x8322 + z8323 = x8323 + z8324 = x8324 + z8325 = x8325 + z8326 = x8326 + z8327 = x8327 + z8328 = x8328 + z8329 = x8329 + z8330 = x8330 + z8331 = x8331 + z8332 = x8332 + z8333 = x8333 + z8334 = x8334 + z8335 = x8335 + z8336 = x8336 + z8337 = x8337 + z8338 = x8338 + z8339 = x8339 + z8340 = x8340 + z8341 = x8341 + z8342 = x8342 + z8343 = x8343 + z8344 = x8344 + z8345 = x8345 + z8346 = x8346 + z8347 = x8347 + z8348 = x8348 + z8349 = x8349 + z8350 = x8350 + z8351 = x8351 + z8352 = x8352 + z8353 = x8353 + z8354 = x8354 + z8355 = x8355 + z8356 = x8356 + z8357 = x8357 + z8358 = x8358 + z8359 = x8359 + z8360 = x8360 + z8361 = x8361 + z8362 = x8362 + z8363 = x8363 + z8364 = x8364 + z8365 = x8365 + z8366 = x8366 + z8367 = x8367 + z8368 = x8368 + z8369 = x8369 + z8370 = x8370 + z8371 = x8371 + z8372 = x8372 + z8373 = x8373 + z8374 = x8374 + z8375 = x8375 + z8376 = x8376 + z8377 = x8377 + z8378 = x8378 + z8379 = x8379 + z8380 = x8380 + z8381 = x8381 + z8382 = x8382 + z8383 = x8383 + z8384 = x8384 + z8385 = x8385 + z8386 = x8386 + z8387 = x8387 + z8388 = x8388 + z8389 = x8389 + z8390 = x8390 + z8391 = x8391 + z8392 = x8392 + z8393 = x8393 + z8394 = x8394 + z8395 = x8395 + z8396 = x8396 + z8397 = x8397 + z8398 = x8398 + z8399 = x8399 + z8400 = x8400 + z8401 = x8401 + z8402 = x8402 + z8403 = x8403 + z8404 = x8404 + z8405 = x8405 + z8406 = x8406 + z8407 = x8407 + z8408 = x8408 + z8409 = x8409 + z8410 = x8410 + z8411 = x8411 + z8412 = x8412 + z8413 = x8413 + z8414 = x8414 + z8415 = x8415 + z8416 = x8416 + z8417 = x8417 + z8418 = x8418 + z8419 = x8419 + z8420 = x8420 + z8421 = x8421 + z8422 = x8422 + z8423 = x8423 + z8424 = x8424 + z8425 = x8425 + z8426 = x8426 + z8427 = x8427 + z8428 = x8428 + z8429 = x8429 + z8430 = x8430 + z8431 = x8431 + z8432 = x8432 + z8433 = x8433 + z8434 = x8434 + z8435 = x8435 + z8436 = x8436 + z8437 = x8437 + z8438 = x8438 + z8439 = x8439 + z8440 = x8440 + z8441 = x8441 + z8442 = x8442 + z8443 = x8443 + z8444 = x8444 + z8445 = x8445 + z8446 = x8446 + z8447 = x8447 + z8448 = x8448 + z8449 = x8449 + z8450 = x8450 + z8451 = x8451 + z8452 = x8452 + z8453 = x8453 + z8454 = x8454 + z8455 = x8455 + z8456 = x8456 + z8457 = x8457 + z8458 = x8458 + z8459 = x8459 + z8460 = x8460 + z8461 = x8461 + z8462 = x8462 + z8463 = x8463 + z8464 = x8464 + z8465 = x8465 + z8466 = x8466 + z8467 = x8467 + z8468 = x8468 + z8469 = x8469 + z8470 = x8470 + z8471 = x8471 + z8472 = x8472 + z8473 = x8473 + z8474 = x8474 + z8475 = x8475 + z8476 = x8476 + z8477 = x8477 + z8478 = x8478 + z8479 = x8479 + z8480 = x8480 + z8481 = x8481 + z8482 = x8482 + z8483 = x8483 + z8484 = x8484 + z8485 = x8485 + z8486 = x8486 + z8487 = x8487 + z8488 = x8488 + z8489 = x8489 + z8490 = x8490 + z8491 = x8491 + z8492 = x8492 + z8493 = x8493 + z8494 = x8494 + z8495 = x8495 + z8496 = x8496 + z8497 = x8497 + z8498 = x8498 + z8499 = x8499 + z8500 = x8500 + z8501 = x8501 + z8502 = x8502 + z8503 = x8503 + z8504 = x8504 + z8505 = x8505 + z8506 = x8506 + z8507 = x8507 + z8508 = x8508 + z8509 = x8509 + z8510 = x8510 + z8511 = x8511 + z8512 = x8512 + z8513 = x8513 + z8514 = x8514 + z8515 = x8515 + z8516 = x8516 + z8517 = x8517 + z8518 = x8518 + z8519 = x8519 + z8520 = x8520 + z8521 = x8521 + z8522 = x8522 + z8523 = x8523 + z8524 = x8524 + z8525 = x8525 + z8526 = x8526 + z8527 = x8527 + z8528 = x8528 + z8529 = x8529 + z8530 = x8530 + z8531 = x8531 + z8532 = x8532 + z8533 = x8533 + z8534 = x8534 + z8535 = x8535 + z8536 = x8536 + z8537 = x8537 + z8538 = x8538 + z8539 = x8539 + z8540 = x8540 + z8541 = x8541 + z8542 = x8542 + z8543 = x8543 + z8544 = x8544 + z8545 = x8545 + z8546 = x8546 + z8547 = x8547 + z8548 = x8548 + z8549 = x8549 + z8550 = x8550 + z8551 = x8551 + z8552 = x8552 + z8553 = x8553 + z8554 = x8554 + z8555 = x8555 + z8556 = x8556 + z8557 = x8557 + z8558 = x8558 + z8559 = x8559 + z8560 = x8560 + z8561 = x8561 + z8562 = x8562 + z8563 = x8563 + z8564 = x8564 + z8565 = x8565 + z8566 = x8566 + z8567 = x8567 + z8568 = x8568 + z8569 = x8569 + z8570 = x8570 + z8571 = x8571 + z8572 = x8572 + z8573 = x8573 + z8574 = x8574 + z8575 = x8575 + z8576 = x8576 + z8577 = x8577 + z8578 = x8578 + z8579 = x8579 + z8580 = x8580 + z8581 = x8581 + z8582 = x8582 + z8583 = x8583 + z8584 = x8584 + z8585 = x8585 + z8586 = x8586 + z8587 = x8587 + z8588 = x8588 + z8589 = x8589 + z8590 = x8590 + z8591 = x8591 + z8592 = x8592 + z8593 = x8593 + z8594 = x8594 + z8595 = x8595 + z8596 = x8596 + z8597 = x8597 + z8598 = x8598 + z8599 = x8599 + z8600 = x8600 + z8601 = x8601 + z8602 = x8602 + z8603 = x8603 + z8604 = x8604 + z8605 = x8605 + z8606 = x8606 + z8607 = x8607 + z8608 = x8608 + z8609 = x8609 + z8610 = x8610 + z8611 = x8611 + z8612 = x8612 + z8613 = x8613 + z8614 = x8614 + z8615 = x8615 + z8616 = x8616 + z8617 = x8617 + z8618 = x8618 + z8619 = x8619 + z8620 = x8620 + z8621 = x8621 + z8622 = x8622 + z8623 = x8623 + z8624 = x8624 + z8625 = x8625 + z8626 = x8626 + z8627 = x8627 + z8628 = x8628 + z8629 = x8629 + z8630 = x8630 + z8631 = x8631 + z8632 = x8632 + z8633 = x8633 + z8634 = x8634 + z8635 = x8635 + z8636 = x8636 + z8637 = x8637 + z8638 = x8638 + z8639 = x8639 + z8640 = x8640 + z8641 = x8641 + z8642 = x8642 + z8643 = x8643 + z8644 = x8644 + z8645 = x8645 + z8646 = x8646 + z8647 = x8647 + z8648 = x8648 + z8649 = x8649 + z8650 = x8650 + z8651 = x8651 + z8652 = x8652 + z8653 = x8653 + z8654 = x8654 + z8655 = x8655 + z8656 = x8656 + z8657 = x8657 + z8658 = x8658 + z8659 = x8659 + z8660 = x8660 + z8661 = x8661 + z8662 = x8662 + z8663 = x8663 + z8664 = x8664 + z8665 = x8665 + z8666 = x8666 + z8667 = x8667 + z8668 = x8668 + z8669 = x8669 + z8670 = x8670 + z8671 = x8671 + z8672 = x8672 + z8673 = x8673 + z8674 = x8674 + z8675 = x8675 + z8676 = x8676 + z8677 = x8677 + z8678 = x8678 + z8679 = x8679 + z8680 = x8680 + z8681 = x8681 + z8682 = x8682 + z8683 = x8683 + z8684 = x8684 + z8685 = x8685 + z8686 = x8686 + z8687 = x8687 + z8688 = x8688 + z8689 = x8689 + z8690 = x8690 + z8691 = x8691 + z8692 = x8692 + z8693 = x8693 + z8694 = x8694 + z8695 = x8695 + z8696 = x8696 + z8697 = x8697 + z8698 = x8698 + z8699 = x8699 + z8700 = x8700 + z8701 = x8701 + z8702 = x8702 + z8703 = x8703 + z8704 = x8704 + z8705 = x8705 + z8706 = x8706 + z8707 = x8707 + z8708 = x8708 + z8709 = x8709 + z8710 = x8710 + z8711 = x8711 + z8712 = x8712 + z8713 = x8713 + z8714 = x8714 + z8715 = x8715 + z8716 = x8716 + z8717 = x8717 + z8718 = x8718 + z8719 = x8719 + z8720 = x8720 + z8721 = x8721 + z8722 = x8722 + z8723 = x8723 + z8724 = x8724 + z8725 = x8725 + z8726 = x8726 + z8727 = x8727 + z8728 = x8728 + z8729 = x8729 + z8730 = x8730 + z8731 = x8731 + z8732 = x8732 + z8733 = x8733 + z8734 = x8734 + z8735 = x8735 + z8736 = x8736 + z8737 = x8737 + z8738 = x8738 + z8739 = x8739 + z8740 = x8740 + z8741 = x8741 + z8742 = x8742 + z8743 = x8743 + z8744 = x8744 + z8745 = x8745 + z8746 = x8746 + z8747 = x8747 + z8748 = x8748 + z8749 = x8749 + z8750 = x8750 + z8751 = x8751 + z8752 = x8752 + z8753 = x8753 + z8754 = x8754 + z8755 = x8755 + z8756 = x8756 + z8757 = x8757 + z8758 = x8758 + z8759 = x8759 + z8760 = x8760 + z8761 = x8761 + z8762 = x8762 + z8763 = x8763 + z8764 = x8764 + z8765 = x8765 + z8766 = x8766 + z8767 = x8767 + z8768 = x8768 + z8769 = x8769 + z8770 = x8770 + z8771 = x8771 + z8772 = x8772 + z8773 = x8773 + z8774 = x8774 + z8775 = x8775 + z8776 = x8776 + z8777 = x8777 + z8778 = x8778 + z8779 = x8779 + z8780 = x8780 + z8781 = x8781 + z8782 = x8782 + z8783 = x8783 + z8784 = x8784 + z8785 = x8785 + z8786 = x8786 + z8787 = x8787 + z8788 = x8788 + z8789 = x8789 + z8790 = x8790 + z8791 = x8791 + z8792 = x8792 + z8793 = x8793 + z8794 = x8794 + z8795 = x8795 + z8796 = x8796 + z8797 = x8797 + z8798 = x8798 + z8799 = x8799 + z8800 = x8800 + z8801 = x8801 + z8802 = x8802 + z8803 = x8803 + z8804 = x8804 + z8805 = x8805 + z8806 = x8806 + z8807 = x8807 + z8808 = x8808 + z8809 = x8809 + z8810 = x8810 + z8811 = x8811 + z8812 = x8812 + z8813 = x8813 + z8814 = x8814 + z8815 = x8815 + z8816 = x8816 + z8817 = x8817 + z8818 = x8818 + z8819 = x8819 + z8820 = x8820 + z8821 = x8821 + z8822 = x8822 + z8823 = x8823 + z8824 = x8824 + z8825 = x8825 + z8826 = x8826 + z8827 = x8827 + z8828 = x8828 + z8829 = x8829 + z8830 = x8830 + z8831 = x8831 + z8832 = x8832 + z8833 = x8833 + z8834 = x8834 + z8835 = x8835 + z8836 = x8836 + z8837 = x8837 + z8838 = x8838 + z8839 = x8839 + z8840 = x8840 + z8841 = x8841 + z8842 = x8842 + z8843 = x8843 + z8844 = x8844 + z8845 = x8845 + z8846 = x8846 + z8847 = x8847 + z8848 = x8848 + z8849 = x8849 + z8850 = x8850 + z8851 = x8851 + z8852 = x8852 + z8853 = x8853 + z8854 = x8854 + z8855 = x8855 + z8856 = x8856 + z8857 = x8857 + z8858 = x8858 + z8859 = x8859 + z8860 = x8860 + z8861 = x8861 + z8862 = x8862 + z8863 = x8863 + z8864 = x8864 + z8865 = x8865 + z8866 = x8866 + z8867 = x8867 + z8868 = x8868 + z8869 = x8869 + z8870 = x8870 + z8871 = x8871 + z8872 = x8872 + z8873 = x8873 + z8874 = x8874 + z8875 = x8875 + z8876 = x8876 + z8877 = x8877 + z8878 = x8878 + z8879 = x8879 + z8880 = x8880 + z8881 = x8881 + z8882 = x8882 + z8883 = x8883 + z8884 = x8884 + z8885 = x8885 + z8886 = x8886 + z8887 = x8887 + z8888 = x8888 + z8889 = x8889 + z8890 = x8890 + z8891 = x8891 + z8892 = x8892 + z8893 = x8893 + z8894 = x8894 + z8895 = x8895 + z8896 = x8896 + z8897 = x8897 + z8898 = x8898 + z8899 = x8899 + z8900 = x8900 + z8901 = x8901 + z8902 = x8902 + z8903 = x8903 + z8904 = x8904 + z8905 = x8905 + z8906 = x8906 + z8907 = x8907 + z8908 = x8908 + z8909 = x8909 + z8910 = x8910 + z8911 = x8911 + z8912 = x8912 + z8913 = x8913 + z8914 = x8914 + z8915 = x8915 + z8916 = x8916 + z8917 = x8917 + z8918 = x8918 + z8919 = x8919 + z8920 = x8920 + z8921 = x8921 + z8922 = x8922 + z8923 = x8923 + z8924 = x8924 + z8925 = x8925 + z8926 = x8926 + z8927 = x8927 + z8928 = x8928 + z8929 = x8929 + z8930 = x8930 + z8931 = x8931 + z8932 = x8932 + z8933 = x8933 + z8934 = x8934 + z8935 = x8935 + z8936 = x8936 + z8937 = x8937 + z8938 = x8938 + z8939 = x8939 + z8940 = x8940 + z8941 = x8941 + z8942 = x8942 + z8943 = x8943 + z8944 = x8944 + z8945 = x8945 + z8946 = x8946 + z8947 = x8947 + z8948 = x8948 + z8949 = x8949 + z8950 = x8950 + z8951 = x8951 + z8952 = x8952 + z8953 = x8953 + z8954 = x8954 + z8955 = x8955 + z8956 = x8956 + z8957 = x8957 + z8958 = x8958 + z8959 = x8959 + z8960 = x8960 + z8961 = x8961 + z8962 = x8962 + z8963 = x8963 + z8964 = x8964 + z8965 = x8965 + z8966 = x8966 + z8967 = x8967 + z8968 = x8968 + z8969 = x8969 + z8970 = x8970 + z8971 = x8971 + z8972 = x8972 + z8973 = x8973 + z8974 = x8974 + z8975 = x8975 + z8976 = x8976 + z8977 = x8977 + z8978 = x8978 + z8979 = x8979 + z8980 = x8980 + z8981 = x8981 + z8982 = x8982 + z8983 = x8983 + z8984 = x8984 + z8985 = x8985 + z8986 = x8986 + z8987 = x8987 + z8988 = x8988 + z8989 = x8989 + z8990 = x8990 + z8991 = x8991 + z8992 = x8992 + z8993 = x8993 + z8994 = x8994 + z8995 = x8995 + z8996 = x8996 + z8997 = x8997 + z8998 = x8998 + z8999 = x8999 + z9000 = x9000 + z9001 = x9001 + z9002 = x9002 + z9003 = x9003 + z9004 = x9004 + z9005 = x9005 + z9006 = x9006 + z9007 = x9007 + z9008 = x9008 + z9009 = x9009 + z9010 = x9010 + z9011 = x9011 + z9012 = x9012 + z9013 = x9013 + z9014 = x9014 + z9015 = x9015 + z9016 = x9016 + z9017 = x9017 + z9018 = x9018 + z9019 = x9019 + z9020 = x9020 + z9021 = x9021 + z9022 = x9022 + z9023 = x9023 + z9024 = x9024 + z9025 = x9025 + z9026 = x9026 + z9027 = x9027 + z9028 = x9028 + z9029 = x9029 + z9030 = x9030 + z9031 = x9031 + z9032 = x9032 + z9033 = x9033 + z9034 = x9034 + z9035 = x9035 + z9036 = x9036 + z9037 = x9037 + z9038 = x9038 + z9039 = x9039 + z9040 = x9040 + z9041 = x9041 + z9042 = x9042 + z9043 = x9043 + z9044 = x9044 + z9045 = x9045 + z9046 = x9046 + z9047 = x9047 + z9048 = x9048 + z9049 = x9049 + z9050 = x9050 + z9051 = x9051 + z9052 = x9052 + z9053 = x9053 + z9054 = x9054 + z9055 = x9055 + z9056 = x9056 + z9057 = x9057 + z9058 = x9058 + z9059 = x9059 + z9060 = x9060 + z9061 = x9061 + z9062 = x9062 + z9063 = x9063 + z9064 = x9064 + z9065 = x9065 + z9066 = x9066 + z9067 = x9067 + z9068 = x9068 + z9069 = x9069 + z9070 = x9070 + z9071 = x9071 + z9072 = x9072 + z9073 = x9073 + z9074 = x9074 + z9075 = x9075 + z9076 = x9076 + z9077 = x9077 + z9078 = x9078 + z9079 = x9079 + z9080 = x9080 + z9081 = x9081 + z9082 = x9082 + z9083 = x9083 + z9084 = x9084 + z9085 = x9085 + z9086 = x9086 + z9087 = x9087 + z9088 = x9088 + z9089 = x9089 + z9090 = x9090 + z9091 = x9091 + z9092 = x9092 + z9093 = x9093 + z9094 = x9094 + z9095 = x9095 + z9096 = x9096 + z9097 = x9097 + z9098 = x9098 + z9099 = x9099 + z9100 = x9100 + z9101 = x9101 + z9102 = x9102 + z9103 = x9103 + z9104 = x9104 + z9105 = x9105 + z9106 = x9106 + z9107 = x9107 + z9108 = x9108 + z9109 = x9109 + z9110 = x9110 + z9111 = x9111 + z9112 = x9112 + z9113 = x9113 + z9114 = x9114 + z9115 = x9115 + z9116 = x9116 + z9117 = x9117 + z9118 = x9118 + z9119 = x9119 + z9120 = x9120 + z9121 = x9121 + z9122 = x9122 + z9123 = x9123 + z9124 = x9124 + z9125 = x9125 + z9126 = x9126 + z9127 = x9127 + z9128 = x9128 + z9129 = x9129 + z9130 = x9130 + z9131 = x9131 + z9132 = x9132 + z9133 = x9133 + z9134 = x9134 + z9135 = x9135 + z9136 = x9136 + z9137 = x9137 + z9138 = x9138 + z9139 = x9139 + z9140 = x9140 + z9141 = x9141 + z9142 = x9142 + z9143 = x9143 + z9144 = x9144 + z9145 = x9145 + z9146 = x9146 + z9147 = x9147 + z9148 = x9148 + z9149 = x9149 + z9150 = x9150 + z9151 = x9151 + z9152 = x9152 + z9153 = x9153 + z9154 = x9154 + z9155 = x9155 + z9156 = x9156 + z9157 = x9157 + z9158 = x9158 + z9159 = x9159 + z9160 = x9160 + z9161 = x9161 + z9162 = x9162 + z9163 = x9163 + z9164 = x9164 + z9165 = x9165 + z9166 = x9166 + z9167 = x9167 + z9168 = x9168 + z9169 = x9169 + z9170 = x9170 + z9171 = x9171 + z9172 = x9172 + z9173 = x9173 + z9174 = x9174 + z9175 = x9175 + z9176 = x9176 + z9177 = x9177 + z9178 = x9178 + z9179 = x9179 + z9180 = x9180 + z9181 = x9181 + z9182 = x9182 + z9183 = x9183 + z9184 = x9184 + z9185 = x9185 + z9186 = x9186 + z9187 = x9187 + z9188 = x9188 + z9189 = x9189 + z9190 = x9190 + z9191 = x9191 + z9192 = x9192 + z9193 = x9193 + z9194 = x9194 + z9195 = x9195 + z9196 = x9196 + z9197 = x9197 + z9198 = x9198 + z9199 = x9199 + z9200 = x9200 + z9201 = x9201 + z9202 = x9202 + z9203 = x9203 + z9204 = x9204 + z9205 = x9205 + z9206 = x9206 + z9207 = x9207 + z9208 = x9208 + z9209 = x9209 + z9210 = x9210 + z9211 = x9211 + z9212 = x9212 + z9213 = x9213 + z9214 = x9214 + z9215 = x9215 + z9216 = x9216 + z9217 = x9217 + z9218 = x9218 + z9219 = x9219 + z9220 = x9220 + z9221 = x9221 + z9222 = x9222 + z9223 = x9223 + z9224 = x9224 + z9225 = x9225 + z9226 = x9226 + z9227 = x9227 + z9228 = x9228 + z9229 = x9229 + z9230 = x9230 + z9231 = x9231 + z9232 = x9232 + z9233 = x9233 + z9234 = x9234 + z9235 = x9235 + z9236 = x9236 + z9237 = x9237 + z9238 = x9238 + z9239 = x9239 + z9240 = x9240 + z9241 = x9241 + z9242 = x9242 + z9243 = x9243 + z9244 = x9244 + z9245 = x9245 + z9246 = x9246 + z9247 = x9247 + z9248 = x9248 + z9249 = x9249 + z9250 = x9250 + z9251 = x9251 + z9252 = x9252 + z9253 = x9253 + z9254 = x9254 + z9255 = x9255 + z9256 = x9256 + z9257 = x9257 + z9258 = x9258 + z9259 = x9259 + z9260 = x9260 + z9261 = x9261 + z9262 = x9262 + z9263 = x9263 + z9264 = x9264 + z9265 = x9265 + z9266 = x9266 + z9267 = x9267 + z9268 = x9268 + z9269 = x9269 + z9270 = x9270 + z9271 = x9271 + z9272 = x9272 + z9273 = x9273 + z9274 = x9274 + z9275 = x9275 + z9276 = x9276 + z9277 = x9277 + z9278 = x9278 + z9279 = x9279 + z9280 = x9280 + z9281 = x9281 + z9282 = x9282 + z9283 = x9283 + z9284 = x9284 + z9285 = x9285 + z9286 = x9286 + z9287 = x9287 + z9288 = x9288 + z9289 = x9289 + z9290 = x9290 + z9291 = x9291 + z9292 = x9292 + z9293 = x9293 + z9294 = x9294 + z9295 = x9295 + z9296 = x9296 + z9297 = x9297 + z9298 = x9298 + z9299 = x9299 + z9300 = x9300 + z9301 = x9301 + z9302 = x9302 + z9303 = x9303 + z9304 = x9304 + z9305 = x9305 + z9306 = x9306 + z9307 = x9307 + z9308 = x9308 + z9309 = x9309 + z9310 = x9310 + z9311 = x9311 + z9312 = x9312 + z9313 = x9313 + z9314 = x9314 + z9315 = x9315 + z9316 = x9316 + z9317 = x9317 + z9318 = x9318 + z9319 = x9319 + z9320 = x9320 + z9321 = x9321 + z9322 = x9322 + z9323 = x9323 + z9324 = x9324 + z9325 = x9325 + z9326 = x9326 + z9327 = x9327 + z9328 = x9328 + z9329 = x9329 + z9330 = x9330 + z9331 = x9331 + z9332 = x9332 + z9333 = x9333 + z9334 = x9334 + z9335 = x9335 + z9336 = x9336 + z9337 = x9337 + z9338 = x9338 + z9339 = x9339 + z9340 = x9340 + z9341 = x9341 + z9342 = x9342 + z9343 = x9343 + z9344 = x9344 + z9345 = x9345 + z9346 = x9346 + z9347 = x9347 + z9348 = x9348 + z9349 = x9349 + z9350 = x9350 + z9351 = x9351 + z9352 = x9352 + z9353 = x9353 + z9354 = x9354 + z9355 = x9355 + z9356 = x9356 + z9357 = x9357 + z9358 = x9358 + z9359 = x9359 + z9360 = x9360 + z9361 = x9361 + z9362 = x9362 + z9363 = x9363 + z9364 = x9364 + z9365 = x9365 + z9366 = x9366 + z9367 = x9367 + z9368 = x9368 + z9369 = x9369 + z9370 = x9370 + z9371 = x9371 + z9372 = x9372 + z9373 = x9373 + z9374 = x9374 + z9375 = x9375 + z9376 = x9376 + z9377 = x9377 + z9378 = x9378 + z9379 = x9379 + z9380 = x9380 + z9381 = x9381 + z9382 = x9382 + z9383 = x9383 + z9384 = x9384 + z9385 = x9385 + z9386 = x9386 + z9387 = x9387 + z9388 = x9388 + z9389 = x9389 + z9390 = x9390 + z9391 = x9391 + z9392 = x9392 + z9393 = x9393 + z9394 = x9394 + z9395 = x9395 + z9396 = x9396 + z9397 = x9397 + z9398 = x9398 + z9399 = x9399 + z9400 = x9400 + z9401 = x9401 + z9402 = x9402 + z9403 = x9403 + z9404 = x9404 + z9405 = x9405 + z9406 = x9406 + z9407 = x9407 + z9408 = x9408 + z9409 = x9409 + z9410 = x9410 + z9411 = x9411 + z9412 = x9412 + z9413 = x9413 + z9414 = x9414 + z9415 = x9415 + z9416 = x9416 + z9417 = x9417 + z9418 = x9418 + z9419 = x9419 + z9420 = x9420 + z9421 = x9421 + z9422 = x9422 + z9423 = x9423 + z9424 = x9424 + z9425 = x9425 + z9426 = x9426 + z9427 = x9427 + z9428 = x9428 + z9429 = x9429 + z9430 = x9430 + z9431 = x9431 + z9432 = x9432 + z9433 = x9433 + z9434 = x9434 + z9435 = x9435 + z9436 = x9436 + z9437 = x9437 + z9438 = x9438 + z9439 = x9439 + z9440 = x9440 + z9441 = x9441 + z9442 = x9442 + z9443 = x9443 + z9444 = x9444 + z9445 = x9445 + z9446 = x9446 + z9447 = x9447 + z9448 = x9448 + z9449 = x9449 + z9450 = x9450 + z9451 = x9451 + z9452 = x9452 + z9453 = x9453 + z9454 = x9454 + z9455 = x9455 + z9456 = x9456 + z9457 = x9457 + z9458 = x9458 + z9459 = x9459 + z9460 = x9460 + z9461 = x9461 + z9462 = x9462 + z9463 = x9463 + z9464 = x9464 + z9465 = x9465 + z9466 = x9466 + z9467 = x9467 + z9468 = x9468 + z9469 = x9469 + z9470 = x9470 + z9471 = x9471 + z9472 = x9472 + z9473 = x9473 + z9474 = x9474 + z9475 = x9475 + z9476 = x9476 + z9477 = x9477 + z9478 = x9478 + z9479 = x9479 + z9480 = x9480 + z9481 = x9481 + z9482 = x9482 + z9483 = x9483 + z9484 = x9484 + z9485 = x9485 + z9486 = x9486 + z9487 = x9487 + z9488 = x9488 + z9489 = x9489 + z9490 = x9490 + z9491 = x9491 + z9492 = x9492 + z9493 = x9493 + z9494 = x9494 + z9495 = x9495 + z9496 = x9496 + z9497 = x9497 + z9498 = x9498 + z9499 = x9499 + z9500 = x9500 + z9501 = x9501 + z9502 = x9502 + z9503 = x9503 + z9504 = x9504 + z9505 = x9505 + z9506 = x9506 + z9507 = x9507 + z9508 = x9508 + z9509 = x9509 + z9510 = x9510 + z9511 = x9511 + z9512 = x9512 + z9513 = x9513 + z9514 = x9514 + z9515 = x9515 + z9516 = x9516 + z9517 = x9517 + z9518 = x9518 + z9519 = x9519 + z9520 = x9520 + z9521 = x9521 + z9522 = x9522 + z9523 = x9523 + z9524 = x9524 + z9525 = x9525 + z9526 = x9526 + z9527 = x9527 + z9528 = x9528 + z9529 = x9529 + z9530 = x9530 + z9531 = x9531 + z9532 = x9532 + z9533 = x9533 + z9534 = x9534 + z9535 = x9535 + z9536 = x9536 + z9537 = x9537 + z9538 = x9538 + z9539 = x9539 + z9540 = x9540 + z9541 = x9541 + z9542 = x9542 + z9543 = x9543 + z9544 = x9544 + z9545 = x9545 + z9546 = x9546 + z9547 = x9547 + z9548 = x9548 + z9549 = x9549 + z9550 = x9550 + z9551 = x9551 + z9552 = x9552 + z9553 = x9553 + z9554 = x9554 + z9555 = x9555 + z9556 = x9556 + z9557 = x9557 + z9558 = x9558 + z9559 = x9559 + z9560 = x9560 + z9561 = x9561 + z9562 = x9562 + z9563 = x9563 + z9564 = x9564 + z9565 = x9565 + z9566 = x9566 + z9567 = x9567 + z9568 = x9568 + z9569 = x9569 + z9570 = x9570 + z9571 = x9571 + z9572 = x9572 + z9573 = x9573 + z9574 = x9574 + z9575 = x9575 + z9576 = x9576 + z9577 = x9577 + z9578 = x9578 + z9579 = x9579 + z9580 = x9580 + z9581 = x9581 + z9582 = x9582 + z9583 = x9583 + z9584 = x9584 + z9585 = x9585 + z9586 = x9586 + z9587 = x9587 + z9588 = x9588 + z9589 = x9589 + z9590 = x9590 + z9591 = x9591 + z9592 = x9592 + z9593 = x9593 + z9594 = x9594 + z9595 = x9595 + z9596 = x9596 + z9597 = x9597 + z9598 = x9598 + z9599 = x9599 + z9600 = x9600 + z9601 = x9601 + z9602 = x9602 + z9603 = x9603 + z9604 = x9604 + z9605 = x9605 + z9606 = x9606 + z9607 = x9607 + z9608 = x9608 + z9609 = x9609 + z9610 = x9610 + z9611 = x9611 + z9612 = x9612 + z9613 = x9613 + z9614 = x9614 + z9615 = x9615 + z9616 = x9616 + z9617 = x9617 + z9618 = x9618 + z9619 = x9619 + z9620 = x9620 + z9621 = x9621 + z9622 = x9622 + z9623 = x9623 + z9624 = x9624 + z9625 = x9625 + z9626 = x9626 + z9627 = x9627 + z9628 = x9628 + z9629 = x9629 + z9630 = x9630 + z9631 = x9631 + z9632 = x9632 + z9633 = x9633 + z9634 = x9634 + z9635 = x9635 + z9636 = x9636 + z9637 = x9637 + z9638 = x9638 + z9639 = x9639 + z9640 = x9640 + z9641 = x9641 + z9642 = x9642 + z9643 = x9643 + z9644 = x9644 + z9645 = x9645 + z9646 = x9646 + z9647 = x9647 + z9648 = x9648 + z9649 = x9649 + z9650 = x9650 + z9651 = x9651 + z9652 = x9652 + z9653 = x9653 + z9654 = x9654 + z9655 = x9655 + z9656 = x9656 + z9657 = x9657 + z9658 = x9658 + z9659 = x9659 + z9660 = x9660 + z9661 = x9661 + z9662 = x9662 + z9663 = x9663 + z9664 = x9664 + z9665 = x9665 + z9666 = x9666 + z9667 = x9667 + z9668 = x9668 + z9669 = x9669 + z9670 = x9670 + z9671 = x9671 + z9672 = x9672 + z9673 = x9673 + z9674 = x9674 + z9675 = x9675 + z9676 = x9676 + z9677 = x9677 + z9678 = x9678 + z9679 = x9679 + z9680 = x9680 + z9681 = x9681 + z9682 = x9682 + z9683 = x9683 + z9684 = x9684 + z9685 = x9685 + z9686 = x9686 + z9687 = x9687 + z9688 = x9688 + z9689 = x9689 + z9690 = x9690 + z9691 = x9691 + z9692 = x9692 + z9693 = x9693 + z9694 = x9694 + z9695 = x9695 + z9696 = x9696 + z9697 = x9697 + z9698 = x9698 + z9699 = x9699 + z9700 = x9700 + z9701 = x9701 + z9702 = x9702 + z9703 = x9703 + z9704 = x9704 + z9705 = x9705 + z9706 = x9706 + z9707 = x9707 + z9708 = x9708 + z9709 = x9709 + z9710 = x9710 + z9711 = x9711 + z9712 = x9712 + z9713 = x9713 + z9714 = x9714 + z9715 = x9715 + z9716 = x9716 + z9717 = x9717 + z9718 = x9718 + z9719 = x9719 + z9720 = x9720 + z9721 = x9721 + z9722 = x9722 + z9723 = x9723 + z9724 = x9724 + z9725 = x9725 + z9726 = x9726 + z9727 = x9727 + z9728 = x9728 + z9729 = x9729 + z9730 = x9730 + z9731 = x9731 + z9732 = x9732 + z9733 = x9733 + z9734 = x9734 + z9735 = x9735 + z9736 = x9736 + z9737 = x9737 + z9738 = x9738 + z9739 = x9739 + z9740 = x9740 + z9741 = x9741 + z9742 = x9742 + z9743 = x9743 + z9744 = x9744 + z9745 = x9745 + z9746 = x9746 + z9747 = x9747 + z9748 = x9748 + z9749 = x9749 + z9750 = x9750 + z9751 = x9751 + z9752 = x9752 + z9753 = x9753 + z9754 = x9754 + z9755 = x9755 + z9756 = x9756 + z9757 = x9757 + z9758 = x9758 + z9759 = x9759 + z9760 = x9760 + z9761 = x9761 + z9762 = x9762 + z9763 = x9763 + z9764 = x9764 + z9765 = x9765 + z9766 = x9766 + z9767 = x9767 + z9768 = x9768 + z9769 = x9769 + z9770 = x9770 + z9771 = x9771 + z9772 = x9772 + z9773 = x9773 + z9774 = x9774 + z9775 = x9775 + z9776 = x9776 + z9777 = x9777 + z9778 = x9778 + z9779 = x9779 + z9780 = x9780 + z9781 = x9781 + z9782 = x9782 + z9783 = x9783 + z9784 = x9784 + z9785 = x9785 + z9786 = x9786 + z9787 = x9787 + z9788 = x9788 + z9789 = x9789 + z9790 = x9790 + z9791 = x9791 + z9792 = x9792 + z9793 = x9793 + z9794 = x9794 + z9795 = x9795 + z9796 = x9796 + z9797 = x9797 + z9798 = x9798 + z9799 = x9799 + z9800 = x9800 + z9801 = x9801 + z9802 = x9802 + z9803 = x9803 + z9804 = x9804 + z9805 = x9805 + z9806 = x9806 + z9807 = x9807 + z9808 = x9808 + z9809 = x9809 + z9810 = x9810 + z9811 = x9811 + z9812 = x9812 + z9813 = x9813 + z9814 = x9814 + z9815 = x9815 + z9816 = x9816 + z9817 = x9817 + z9818 = x9818 + z9819 = x9819 + z9820 = x9820 + z9821 = x9821 + z9822 = x9822 + z9823 = x9823 + z9824 = x9824 + z9825 = x9825 + z9826 = x9826 + z9827 = x9827 + z9828 = x9828 + z9829 = x9829 + z9830 = x9830 + z9831 = x9831 + z9832 = x9832 + z9833 = x9833 + z9834 = x9834 + z9835 = x9835 + z9836 = x9836 + z9837 = x9837 + z9838 = x9838 + z9839 = x9839 + z9840 = x9840 + z9841 = x9841 + z9842 = x9842 + z9843 = x9843 + z9844 = x9844 + z9845 = x9845 + z9846 = x9846 + z9847 = x9847 + z9848 = x9848 + z9849 = x9849 + z9850 = x9850 + z9851 = x9851 + z9852 = x9852 + z9853 = x9853 + z9854 = x9854 + z9855 = x9855 + z9856 = x9856 + z9857 = x9857 + z9858 = x9858 + z9859 = x9859 + z9860 = x9860 + z9861 = x9861 + z9862 = x9862 + z9863 = x9863 + z9864 = x9864 + z9865 = x9865 + z9866 = x9866 + z9867 = x9867 + z9868 = x9868 + z9869 = x9869 + z9870 = x9870 + z9871 = x9871 + z9872 = x9872 + z9873 = x9873 + z9874 = x9874 + z9875 = x9875 + z9876 = x9876 + z9877 = x9877 + z9878 = x9878 + z9879 = x9879 + z9880 = x9880 + z9881 = x9881 + z9882 = x9882 + z9883 = x9883 + z9884 = x9884 + z9885 = x9885 + z9886 = x9886 + z9887 = x9887 + z9888 = x9888 + z9889 = x9889 + z9890 = x9890 + z9891 = x9891 + z9892 = x9892 + z9893 = x9893 + z9894 = x9894 + z9895 = x9895 + z9896 = x9896 + z9897 = x9897 + z9898 = x9898 + z9899 = x9899 + z9900 = x9900 + z9901 = x9901 + z9902 = x9902 + z9903 = x9903 + z9904 = x9904 + z9905 = x9905 + z9906 = x9906 + z9907 = x9907 + z9908 = x9908 + z9909 = x9909 + z9910 = x9910 + z9911 = x9911 + z9912 = x9912 + z9913 = x9913 + z9914 = x9914 + z9915 = x9915 + z9916 = x9916 + z9917 = x9917 + z9918 = x9918 + z9919 = x9919 + z9920 = x9920 + z9921 = x9921 + z9922 = x9922 + z9923 = x9923 + z9924 = x9924 + z9925 = x9925 + z9926 = x9926 + z9927 = x9927 + z9928 = x9928 + z9929 = x9929 + z9930 = x9930 + z9931 = x9931 + z9932 = x9932 + z9933 = x9933 + z9934 = x9934 + z9935 = x9935 + z9936 = x9936 + z9937 = x9937 + z9938 = x9938 + z9939 = x9939 + z9940 = x9940 + z9941 = x9941 + z9942 = x9942 + z9943 = x9943 + z9944 = x9944 + z9945 = x9945 + z9946 = x9946 + z9947 = x9947 + z9948 = x9948 + z9949 = x9949 + z9950 = x9950 + z9951 = x9951 + z9952 = x9952 + z9953 = x9953 + z9954 = x9954 + z9955 = x9955 + z9956 = x9956 + z9957 = x9957 + z9958 = x9958 + z9959 = x9959 + z9960 = x9960 + z9961 = x9961 + z9962 = x9962 + z9963 = x9963 + z9964 = x9964 + z9965 = x9965 + z9966 = x9966 + z9967 = x9967 + z9968 = x9968 + z9969 = x9969 + z9970 = x9970 + z9971 = x9971 + z9972 = x9972 + z9973 = x9973 + z9974 = x9974 + z9975 = x9975 + z9976 = x9976 + z9977 = x9977 + z9978 = x9978 + z9979 = x9979 + z9980 = x9980 + z9981 = x9981 + z9982 = x9982 + z9983 = x9983 + z9984 = x9984 + z9985 = x9985 + z9986 = x9986 + z9987 = x9987 + z9988 = x9988 + z9989 = x9989 + z9990 = x9990 + z9991 = x9991 + z9992 = x9992 + z9993 = x9993 + z9994 = x9994 + z9995 = x9995 + z9996 = x9996 + z9997 = x9997 + z9998 = x9998 + z9999 = x9999 + z10000 = x10000 + z10001 = x10001 + z10002 = x10002 + z10003 = x10003 + z10004 = x10004 + z10005 = x10005 + z10006 = x10006 + z10007 = x10007 + z10008 = x10008 + z10009 = x10009 + z10010 = x10010 + z10011 = x10011 + z10012 = x10012 + z10013 = x10013 + z10014 = x10014 + z10015 = x10015 + z10016 = x10016 + z10017 = x10017 + z10018 = x10018 + z10019 = x10019 + z10020 = x10020 + z10021 = x10021 + z10022 = x10022 + z10023 = x10023 + z10024 = x10024 + z10025 = x10025 + z10026 = x10026 + z10027 = x10027 + z10028 = x10028 + z10029 = x10029 + z10030 = x10030 + z10031 = x10031 + z10032 = x10032 + z10033 = x10033 + z10034 = x10034 + z10035 = x10035 + z10036 = x10036 + z10037 = x10037 + z10038 = x10038 + z10039 = x10039 + z10040 = x10040 + z10041 = x10041 + z10042 = x10042 + z10043 = x10043 + z10044 = x10044 + z10045 = x10045 + z10046 = x10046 + z10047 = x10047 + z10048 = x10048 + z10049 = x10049 + z10050 = x10050 + z10051 = x10051 + z10052 = x10052 + z10053 = x10053 + z10054 = x10054 + z10055 = x10055 + z10056 = x10056 + z10057 = x10057 + z10058 = x10058 + z10059 = x10059 + z10060 = x10060 + z10061 = x10061 + z10062 = x10062 + z10063 = x10063 + z10064 = x10064 + z10065 = x10065 + z10066 = x10066 + z10067 = x10067 + z10068 = x10068 + z10069 = x10069 + z10070 = x10070 + z10071 = x10071 + z10072 = x10072 + z10073 = x10073 + z10074 = x10074 + z10075 = x10075 + z10076 = x10076 + z10077 = x10077 + z10078 = x10078 + z10079 = x10079 + z10080 = x10080 + z10081 = x10081 + z10082 = x10082 + z10083 = x10083 + z10084 = x10084 + z10085 = x10085 + z10086 = x10086 + z10087 = x10087 + z10088 = x10088 + z10089 = x10089 + z10090 = x10090 + z10091 = x10091 + z10092 = x10092 + z10093 = x10093 + z10094 = x10094 + z10095 = x10095 + z10096 = x10096 + z10097 = x10097 + z10098 = x10098 + z10099 = x10099 + z10100 = x10100 + z10101 = x10101 + z10102 = x10102 + z10103 = x10103 + z10104 = x10104 + z10105 = x10105 + z10106 = x10106 + z10107 = x10107 + z10108 = x10108 + z10109 = x10109 + z10110 = x10110 + z10111 = x10111 + z10112 = x10112 + z10113 = x10113 + z10114 = x10114 + z10115 = x10115 + z10116 = x10116 + z10117 = x10117 + z10118 = x10118 + z10119 = x10119 + z10120 = x10120 + z10121 = x10121 + z10122 = x10122 + z10123 = x10123 + z10124 = x10124 + z10125 = x10125 + z10126 = x10126 + z10127 = x10127 + z10128 = x10128 + z10129 = x10129 + z10130 = x10130 + z10131 = x10131 + z10132 = x10132 + z10133 = x10133 + z10134 = x10134 + z10135 = x10135 + z10136 = x10136 + z10137 = x10137 + z10138 = x10138 + z10139 = x10139 + z10140 = x10140 + z10141 = x10141 + z10142 = x10142 + z10143 = x10143 + z10144 = x10144 + z10145 = x10145 + z10146 = x10146 + z10147 = x10147 + z10148 = x10148 + z10149 = x10149 + z10150 = x10150 + z10151 = x10151 + z10152 = x10152 + z10153 = x10153 + z10154 = x10154 + z10155 = x10155 + z10156 = x10156 + z10157 = x10157 + z10158 = x10158 + z10159 = x10159 + z10160 = x10160 + z10161 = x10161 + z10162 = x10162 + z10163 = x10163 + z10164 = x10164 + z10165 = x10165 + z10166 = x10166 + z10167 = x10167 + z10168 = x10168 + z10169 = x10169 + z10170 = x10170 + z10171 = x10171 + z10172 = x10172 + z10173 = x10173 + z10174 = x10174 + z10175 = x10175 + z10176 = x10176 + z10177 = x10177 + z10178 = x10178 + z10179 = x10179 + z10180 = x10180 + z10181 = x10181 + z10182 = x10182 + z10183 = x10183 + z10184 = x10184 + z10185 = x10185 + z10186 = x10186 + z10187 = x10187 + z10188 = x10188 + z10189 = x10189 + z10190 = x10190 + z10191 = x10191 + z10192 = x10192 + z10193 = x10193 + z10194 = x10194 + z10195 = x10195 + z10196 = x10196 + z10197 = x10197 + z10198 = x10198 + z10199 = x10199 + z10200 = x10200 + z10201 = x10201 + z10202 = x10202 + z10203 = x10203 + z10204 = x10204 + z10205 = x10205 + z10206 = x10206 + z10207 = x10207 + z10208 = x10208 + z10209 = x10209 + z10210 = x10210 + z10211 = x10211 + z10212 = x10212 + z10213 = x10213 + z10214 = x10214 + z10215 = x10215 + z10216 = x10216 + z10217 = x10217 + z10218 = x10218 + z10219 = x10219 + z10220 = x10220 + z10221 = x10221 + z10222 = x10222 + z10223 = x10223 + z10224 = x10224 + z10225 = x10225 + z10226 = x10226 + z10227 = x10227 + z10228 = x10228 + z10229 = x10229 + z10230 = x10230 + z10231 = x10231 + z10232 = x10232 + z10233 = x10233 + z10234 = x10234 + z10235 = x10235 + z10236 = x10236 + z10237 = x10237 + z10238 = x10238 + z10239 = x10239 + z10240 = x10240 + z10241 = x10241 + z10242 = x10242 + z10243 = x10243 + z10244 = x10244 + z10245 = x10245 + z10246 = x10246 + z10247 = x10247 + z10248 = x10248 + z10249 = x10249 + z10250 = x10250 + z10251 = x10251 + z10252 = x10252 + z10253 = x10253 + z10254 = x10254 + z10255 = x10255 + z10256 = x10256 + z10257 = x10257 + z10258 = x10258 + z10259 = x10259 + z10260 = x10260 + z10261 = x10261 + z10262 = x10262 + z10263 = x10263 + z10264 = x10264 + z10265 = x10265 + z10266 = x10266 + z10267 = x10267 + z10268 = x10268 + z10269 = x10269 + z10270 = x10270 + z10271 = x10271 + z10272 = x10272 + z10273 = x10273 + z10274 = x10274 + z10275 = x10275 + z10276 = x10276 + z10277 = x10277 + z10278 = x10278 + z10279 = x10279 + z10280 = x10280 + z10281 = x10281 + z10282 = x10282 + z10283 = x10283 + z10284 = x10284 + z10285 = x10285 + z10286 = x10286 + z10287 = x10287 + z10288 = x10288 + z10289 = x10289 + z10290 = x10290 + z10291 = x10291 + z10292 = x10292 + z10293 = x10293 + z10294 = x10294 + z10295 = x10295 + z10296 = x10296 + z10297 = x10297 + z10298 = x10298 + z10299 = x10299 + z10300 = x10300 + z10301 = x10301 + z10302 = x10302 + z10303 = x10303 + z10304 = x10304 + z10305 = x10305 + z10306 = x10306 + z10307 = x10307 + z10308 = x10308 + z10309 = x10309 + z10310 = x10310 + z10311 = x10311 + z10312 = x10312 + z10313 = x10313 + z10314 = x10314 + z10315 = x10315 + z10316 = x10316 + z10317 = x10317 + z10318 = x10318 + z10319 = x10319 + z10320 = x10320 + z10321 = x10321 + z10322 = x10322 + z10323 = x10323 + z10324 = x10324 + z10325 = x10325 + z10326 = x10326 + z10327 = x10327 + z10328 = x10328 + z10329 = x10329 + z10330 = x10330 + z10331 = x10331 + z10332 = x10332 + z10333 = x10333 + z10334 = x10334 + z10335 = x10335 + z10336 = x10336 + z10337 = x10337 + z10338 = x10338 + z10339 = x10339 + z10340 = x10340 + z10341 = x10341 + z10342 = x10342 + z10343 = x10343 + z10344 = x10344 + z10345 = x10345 + z10346 = x10346 + z10347 = x10347 + z10348 = x10348 + z10349 = x10349 + z10350 = x10350 + z10351 = x10351 + z10352 = x10352 + z10353 = x10353 + z10354 = x10354 + z10355 = x10355 + z10356 = x10356 + z10357 = x10357 + z10358 = x10358 + z10359 = x10359 + z10360 = x10360 + z10361 = x10361 + z10362 = x10362 + z10363 = x10363 + z10364 = x10364 + z10365 = x10365 + z10366 = x10366 + z10367 = x10367 + z10368 = x10368 + z10369 = x10369 + z10370 = x10370 + z10371 = x10371 + z10372 = x10372 + z10373 = x10373 + z10374 = x10374 + z10375 = x10375 + z10376 = x10376 + z10377 = x10377 + z10378 = x10378 + z10379 = x10379 + z10380 = x10380 + z10381 = x10381 + z10382 = x10382 + z10383 = x10383 + z10384 = x10384 + z10385 = x10385 + z10386 = x10386 + z10387 = x10387 + z10388 = x10388 + z10389 = x10389 + z10390 = x10390 + z10391 = x10391 + z10392 = x10392 + z10393 = x10393 + z10394 = x10394 + z10395 = x10395 + z10396 = x10396 + z10397 = x10397 + z10398 = x10398 + z10399 = x10399 + z10400 = x10400 + z10401 = x10401 + z10402 = x10402 + z10403 = x10403 + z10404 = x10404 + z10405 = x10405 + z10406 = x10406 + z10407 = x10407 + z10408 = x10408 + z10409 = x10409 + z10410 = x10410 + z10411 = x10411 + z10412 = x10412 + z10413 = x10413 + z10414 = x10414 + z10415 = x10415 + z10416 = x10416 + z10417 = x10417 + z10418 = x10418 + z10419 = x10419 + z10420 = x10420 + z10421 = x10421 + z10422 = x10422 + z10423 = x10423 + z10424 = x10424 + z10425 = x10425 + z10426 = x10426 + z10427 = x10427 + z10428 = x10428 + z10429 = x10429 + z10430 = x10430 + z10431 = x10431 + z10432 = x10432 + z10433 = x10433 + z10434 = x10434 + z10435 = x10435 + z10436 = x10436 + z10437 = x10437 + z10438 = x10438 + z10439 = x10439 + z10440 = x10440 + z10441 = x10441 + z10442 = x10442 + z10443 = x10443 + z10444 = x10444 + z10445 = x10445 + z10446 = x10446 + z10447 = x10447 + z10448 = x10448 + z10449 = x10449 + z10450 = x10450 + z10451 = x10451 + z10452 = x10452 + z10453 = x10453 + z10454 = x10454 + z10455 = x10455 + z10456 = x10456 + z10457 = x10457 + z10458 = x10458 + z10459 = x10459 + z10460 = x10460 + z10461 = x10461 + z10462 = x10462 + z10463 = x10463 + z10464 = x10464 + z10465 = x10465 + z10466 = x10466 + z10467 = x10467 + z10468 = x10468 + z10469 = x10469 + z10470 = x10470 + z10471 = x10471 + z10472 = x10472 + z10473 = x10473 + z10474 = x10474 + z10475 = x10475 + z10476 = x10476 + z10477 = x10477 + z10478 = x10478 + z10479 = x10479 + z10480 = x10480 + z10481 = x10481 + z10482 = x10482 + z10483 = x10483 + z10484 = x10484 + z10485 = x10485 + z10486 = x10486 + z10487 = x10487 + z10488 = x10488 + z10489 = x10489 + z10490 = x10490 + z10491 = x10491 + z10492 = x10492 + z10493 = x10493 + z10494 = x10494 + z10495 = x10495 + z10496 = x10496 + z10497 = x10497 + z10498 = x10498 + z10499 = x10499 + z10500 = x10500 + z10501 = x10501 + z10502 = x10502 + z10503 = x10503 + z10504 = x10504 + z10505 = x10505 + z10506 = x10506 + z10507 = x10507 + z10508 = x10508 + z10509 = x10509 + z10510 = x10510 + z10511 = x10511 + z10512 = x10512 + z10513 = x10513 + z10514 = x10514 + z10515 = x10515 + z10516 = x10516 + z10517 = x10517 + z10518 = x10518 + z10519 = x10519 + z10520 = x10520 + z10521 = x10521 + z10522 = x10522 + z10523 = x10523 + z10524 = x10524 + z10525 = x10525 + z10526 = x10526 + z10527 = x10527 + z10528 = x10528 + z10529 = x10529 + z10530 = x10530 + z10531 = x10531 + z10532 = x10532 + z10533 = x10533 + z10534 = x10534 + z10535 = x10535 + z10536 = x10536 + z10537 = x10537 + z10538 = x10538 + z10539 = x10539 + z10540 = x10540 + z10541 = x10541 + z10542 = x10542 + z10543 = x10543 + z10544 = x10544 + z10545 = x10545 + z10546 = x10546 + z10547 = x10547 + z10548 = x10548 + z10549 = x10549 + z10550 = x10550 + z10551 = x10551 + z10552 = x10552 + z10553 = x10553 + z10554 = x10554 + z10555 = x10555 + z10556 = x10556 + z10557 = x10557 + z10558 = x10558 + z10559 = x10559 + z10560 = x10560 + z10561 = x10561 + z10562 = x10562 + z10563 = x10563 + z10564 = x10564 + z10565 = x10565 + z10566 = x10566 + z10567 = x10567 + z10568 = x10568 + z10569 = x10569 + z10570 = x10570 + z10571 = x10571 + z10572 = x10572 + z10573 = x10573 + z10574 = x10574 + z10575 = x10575 + z10576 = x10576 + z10577 = x10577 + z10578 = x10578 + z10579 = x10579 + z10580 = x10580 + z10581 = x10581 + z10582 = x10582 + z10583 = x10583 + z10584 = x10584 + z10585 = x10585 + z10586 = x10586 + z10587 = x10587 + z10588 = x10588 + z10589 = x10589 + z10590 = x10590 + z10591 = x10591 + z10592 = x10592 + z10593 = x10593 + z10594 = x10594 + z10595 = x10595 + z10596 = x10596 + z10597 = x10597 + z10598 = x10598 + z10599 = x10599 + z10600 = x10600 + z10601 = x10601 + z10602 = x10602 + z10603 = x10603 + z10604 = x10604 + z10605 = x10605 + z10606 = x10606 + z10607 = x10607 + z10608 = x10608 + z10609 = x10609 + z10610 = x10610 + z10611 = x10611 + z10612 = x10612 + z10613 = x10613 + z10614 = x10614 + z10615 = x10615 + z10616 = x10616 + z10617 = x10617 + z10618 = x10618 + z10619 = x10619 + z10620 = x10620 + z10621 = x10621 + z10622 = x10622 + z10623 = x10623 + z10624 = x10624 + z10625 = x10625 + z10626 = x10626 + z10627 = x10627 + z10628 = x10628 + z10629 = x10629 + z10630 = x10630 + z10631 = x10631 + z10632 = x10632 + z10633 = x10633 + z10634 = x10634 + z10635 = x10635 + z10636 = x10636 + z10637 = x10637 + z10638 = x10638 + z10639 = x10639 + z10640 = x10640 + z10641 = x10641 + z10642 = x10642 + z10643 = x10643 + z10644 = x10644 + z10645 = x10645 + z10646 = x10646 + z10647 = x10647 + z10648 = x10648 + z10649 = x10649 + z10650 = x10650 + z10651 = x10651 + z10652 = x10652 + z10653 = x10653 + z10654 = x10654 + z10655 = x10655 + z10656 = x10656 + z10657 = x10657 + z10658 = x10658 + z10659 = x10659 + z10660 = x10660 + z10661 = x10661 + z10662 = x10662 + z10663 = x10663 + z10664 = x10664 + z10665 = x10665 + z10666 = x10666 + z10667 = x10667 + z10668 = x10668 + z10669 = x10669 + z10670 = x10670 + z10671 = x10671 + z10672 = x10672 + z10673 = x10673 + z10674 = x10674 + z10675 = x10675 + z10676 = x10676 + z10677 = x10677 + z10678 = x10678 + z10679 = x10679 + z10680 = x10680 + z10681 = x10681 + z10682 = x10682 + z10683 = x10683 + z10684 = x10684 + z10685 = x10685 + z10686 = x10686 + z10687 = x10687 + z10688 = x10688 + z10689 = x10689 + z10690 = x10690 + z10691 = x10691 + z10692 = x10692 + z10693 = x10693 + z10694 = x10694 + z10695 = x10695 + z10696 = x10696 + z10697 = x10697 + z10698 = x10698 + z10699 = x10699 + z10700 = x10700 + z10701 = x10701 + z10702 = x10702 + z10703 = x10703 + z10704 = x10704 + z10705 = x10705 + z10706 = x10706 + z10707 = x10707 + z10708 = x10708 + z10709 = x10709 + z10710 = x10710 + z10711 = x10711 + z10712 = x10712 + z10713 = x10713 + z10714 = x10714 + z10715 = x10715 + z10716 = x10716 + z10717 = x10717 + z10718 = x10718 + z10719 = x10719 + z10720 = x10720 + z10721 = x10721 + z10722 = x10722 + z10723 = x10723 + z10724 = x10724 + z10725 = x10725 + z10726 = x10726 + z10727 = x10727 + z10728 = x10728 + z10729 = x10729 + z10730 = x10730 + z10731 = x10731 + z10732 = x10732 + z10733 = x10733 + z10734 = x10734 + z10735 = x10735 + z10736 = x10736 + z10737 = x10737 + z10738 = x10738 + z10739 = x10739 + z10740 = x10740 + z10741 = x10741 + z10742 = x10742 + z10743 = x10743 + z10744 = x10744 + z10745 = x10745 + z10746 = x10746 + z10747 = x10747 + z10748 = x10748 + z10749 = x10749 + z10750 = x10750 + z10751 = x10751 + z10752 = x10752 + z10753 = x10753 + z10754 = x10754 + z10755 = x10755 + z10756 = x10756 + z10757 = x10757 + z10758 = x10758 + z10759 = x10759 + z10760 = x10760 + z10761 = x10761 + z10762 = x10762 + z10763 = x10763 + z10764 = x10764 + z10765 = x10765 + z10766 = x10766 + z10767 = x10767 + z10768 = x10768 + z10769 = x10769 + z10770 = x10770 + z10771 = x10771 + z10772 = x10772 + z10773 = x10773 + z10774 = x10774 + z10775 = x10775 + z10776 = x10776 + z10777 = x10777 + z10778 = x10778 + z10779 = x10779 + z10780 = x10780 + z10781 = x10781 + z10782 = x10782 + z10783 = x10783 + z10784 = x10784 + z10785 = x10785 + z10786 = x10786 + z10787 = x10787 + z10788 = x10788 + z10789 = x10789 + z10790 = x10790 + z10791 = x10791 + z10792 = x10792 + z10793 = x10793 + z10794 = x10794 + z10795 = x10795 + z10796 = x10796 + z10797 = x10797 + z10798 = x10798 + z10799 = x10799 + z10800 = x10800 + z10801 = x10801 + z10802 = x10802 + z10803 = x10803 + z10804 = x10804 + z10805 = x10805 + z10806 = x10806 + z10807 = x10807 + z10808 = x10808 + z10809 = x10809 + z10810 = x10810 + z10811 = x10811 + z10812 = x10812 + z10813 = x10813 + z10814 = x10814 + z10815 = x10815 + z10816 = x10816 + z10817 = x10817 + z10818 = x10818 + z10819 = x10819 + z10820 = x10820 + z10821 = x10821 + z10822 = x10822 + z10823 = x10823 + z10824 = x10824 + z10825 = x10825 + z10826 = x10826 + z10827 = x10827 + z10828 = x10828 + z10829 = x10829 + z10830 = x10830 + z10831 = x10831 + z10832 = x10832 + z10833 = x10833 + z10834 = x10834 + z10835 = x10835 + z10836 = x10836 + z10837 = x10837 + z10838 = x10838 + z10839 = x10839 + z10840 = x10840 + z10841 = x10841 + z10842 = x10842 + z10843 = x10843 + z10844 = x10844 + z10845 = x10845 + z10846 = x10846 + z10847 = x10847 + z10848 = x10848 + z10849 = x10849 + z10850 = x10850 + z10851 = x10851 + z10852 = x10852 + z10853 = x10853 + z10854 = x10854 + z10855 = x10855 + z10856 = x10856 + z10857 = x10857 + z10858 = x10858 + z10859 = x10859 + z10860 = x10860 + z10861 = x10861 + z10862 = x10862 + z10863 = x10863 + z10864 = x10864 + z10865 = x10865 + z10866 = x10866 + z10867 = x10867 + z10868 = x10868 + z10869 = x10869 + z10870 = x10870 + z10871 = x10871 + z10872 = x10872 + z10873 = x10873 + z10874 = x10874 + z10875 = x10875 + z10876 = x10876 + z10877 = x10877 + z10878 = x10878 + z10879 = x10879 + z10880 = x10880 + z10881 = x10881 + z10882 = x10882 + z10883 = x10883 + z10884 = x10884 + z10885 = x10885 + z10886 = x10886 + z10887 = x10887 + z10888 = x10888 + z10889 = x10889 + z10890 = x10890 + z10891 = x10891 + z10892 = x10892 + z10893 = x10893 + z10894 = x10894 + z10895 = x10895 + z10896 = x10896 + z10897 = x10897 + z10898 = x10898 + z10899 = x10899 + z10900 = x10900 + z10901 = x10901 + z10902 = x10902 + z10903 = x10903 + z10904 = x10904 + z10905 = x10905 + z10906 = x10906 + z10907 = x10907 + z10908 = x10908 + z10909 = x10909 + z10910 = x10910 + z10911 = x10911 + z10912 = x10912 + z10913 = x10913 + z10914 = x10914 + z10915 = x10915 + z10916 = x10916 + z10917 = x10917 + z10918 = x10918 + z10919 = x10919 + z10920 = x10920 + z10921 = x10921 + z10922 = x10922 + z10923 = x10923 + z10924 = x10924 + z10925 = x10925 + z10926 = x10926 + z10927 = x10927 + z10928 = x10928 + z10929 = x10929 + z10930 = x10930 + z10931 = x10931 + z10932 = x10932 + z10933 = x10933 + z10934 = x10934 + z10935 = x10935 + z10936 = x10936 + z10937 = x10937 + z10938 = x10938 + z10939 = x10939 + z10940 = x10940 + z10941 = x10941 + z10942 = x10942 + z10943 = x10943 + z10944 = x10944 + z10945 = x10945 + z10946 = x10946 + z10947 = x10947 + z10948 = x10948 + z10949 = x10949 + z10950 = x10950 + z10951 = x10951 + z10952 = x10952 + z10953 = x10953 + z10954 = x10954 + z10955 = x10955 + z10956 = x10956 + z10957 = x10957 + z10958 = x10958 + z10959 = x10959 + z10960 = x10960 + z10961 = x10961 + z10962 = x10962 + z10963 = x10963 + z10964 = x10964 + z10965 = x10965 + z10966 = x10966 + z10967 = x10967 + z10968 = x10968 + z10969 = x10969 + z10970 = x10970 + z10971 = x10971 + z10972 = x10972 + z10973 = x10973 + z10974 = x10974 + z10975 = x10975 + z10976 = x10976 + z10977 = x10977 + z10978 = x10978 + z10979 = x10979 + z10980 = x10980 + z10981 = x10981 + z10982 = x10982 + z10983 = x10983 + z10984 = x10984 + z10985 = x10985 + z10986 = x10986 + z10987 = x10987 + z10988 = x10988 + z10989 = x10989 + z10990 = x10990 + z10991 = x10991 + z10992 = x10992 + z10993 = x10993 + z10994 = x10994 + z10995 = x10995 + z10996 = x10996 + z10997 = x10997 + z10998 = x10998 + z10999 = x10999 + z11000 = x11000 + z11001 = x11001 + z11002 = x11002 + z11003 = x11003 + z11004 = x11004 + z11005 = x11005 + z11006 = x11006 + z11007 = x11007 + z11008 = x11008 + z11009 = x11009 + z11010 = x11010 + z11011 = x11011 + z11012 = x11012 + z11013 = x11013 + z11014 = x11014 + z11015 = x11015 + z11016 = x11016 + z11017 = x11017 + z11018 = x11018 + z11019 = x11019 + z11020 = x11020 + z11021 = x11021 + z11022 = x11022 + z11023 = x11023 + z11024 = x11024 + z11025 = x11025 + z11026 = x11026 + z11027 = x11027 + z11028 = x11028 + z11029 = x11029 + z11030 = x11030 + z11031 = x11031 + z11032 = x11032 + z11033 = x11033 + z11034 = x11034 + z11035 = x11035 + z11036 = x11036 + z11037 = x11037 + z11038 = x11038 + z11039 = x11039 + z11040 = x11040 + z11041 = x11041 + z11042 = x11042 + z11043 = x11043 + z11044 = x11044 + z11045 = x11045 + z11046 = x11046 + z11047 = x11047 + z11048 = x11048 + z11049 = x11049 + z11050 = x11050 + z11051 = x11051 + z11052 = x11052 + z11053 = x11053 + z11054 = x11054 + z11055 = x11055 + z11056 = x11056 + z11057 = x11057 + z11058 = x11058 + z11059 = x11059 + z11060 = x11060 + z11061 = x11061 + z11062 = x11062 + z11063 = x11063 + z11064 = x11064 + z11065 = x11065 + z11066 = x11066 + z11067 = x11067 + z11068 = x11068 + z11069 = x11069 + z11070 = x11070 + z11071 = x11071 + z11072 = x11072 + z11073 = x11073 + z11074 = x11074 + z11075 = x11075 + z11076 = x11076 + z11077 = x11077 + z11078 = x11078 + z11079 = x11079 + z11080 = x11080 + z11081 = x11081 + z11082 = x11082 + z11083 = x11083 + z11084 = x11084 + z11085 = x11085 + z11086 = x11086 + z11087 = x11087 + z11088 = x11088 + z11089 = x11089 + z11090 = x11090 + z11091 = x11091 + z11092 = x11092 + z11093 = x11093 + z11094 = x11094 + z11095 = x11095 + z11096 = x11096 + z11097 = x11097 + z11098 = x11098 + z11099 = x11099 + z11100 = x11100 + z11101 = x11101 + z11102 = x11102 + z11103 = x11103 + z11104 = x11104 + z11105 = x11105 + z11106 = x11106 + z11107 = x11107 + z11108 = x11108 + z11109 = x11109 + z11110 = x11110 + z11111 = x11111 + z11112 = x11112 + z11113 = x11113 + z11114 = x11114 + z11115 = x11115 + z11116 = x11116 + z11117 = x11117 + z11118 = x11118 + z11119 = x11119 + z11120 = x11120 + z11121 = x11121 + z11122 = x11122 + z11123 = x11123 + z11124 = x11124 + z11125 = x11125 + z11126 = x11126 + z11127 = x11127 + z11128 = x11128 + z11129 = x11129 + z11130 = x11130 + z11131 = x11131 + z11132 = x11132 + z11133 = x11133 + z11134 = x11134 + z11135 = x11135 + z11136 = x11136 + z11137 = x11137 + z11138 = x11138 + z11139 = x11139 + z11140 = x11140 + z11141 = x11141 + z11142 = x11142 + z11143 = x11143 + z11144 = x11144 + z11145 = x11145 + z11146 = x11146 + z11147 = x11147 + z11148 = x11148 + z11149 = x11149 + z11150 = x11150 + z11151 = x11151 + z11152 = x11152 + z11153 = x11153 + z11154 = x11154 + z11155 = x11155 + z11156 = x11156 + z11157 = x11157 + z11158 = x11158 + z11159 = x11159 + z11160 = x11160 + z11161 = x11161 + z11162 = x11162 + z11163 = x11163 + z11164 = x11164 + z11165 = x11165 + z11166 = x11166 + z11167 = x11167 + z11168 = x11168 + z11169 = x11169 + z11170 = x11170 + z11171 = x11171 + z11172 = x11172 + z11173 = x11173 + z11174 = x11174 + z11175 = x11175 + z11176 = x11176 + z11177 = x11177 + z11178 = x11178 + z11179 = x11179 + z11180 = x11180 + z11181 = x11181 + z11182 = x11182 + z11183 = x11183 + z11184 = x11184 + z11185 = x11185 + z11186 = x11186 + z11187 = x11187 + z11188 = x11188 + z11189 = x11189 + z11190 = x11190 + z11191 = x11191 + z11192 = x11192 + z11193 = x11193 + z11194 = x11194 + z11195 = x11195 + z11196 = x11196 + z11197 = x11197 + z11198 = x11198 + z11199 = x11199 + z11200 = x11200 + z11201 = x11201 + z11202 = x11202 + z11203 = x11203 + z11204 = x11204 + z11205 = x11205 + z11206 = x11206 + z11207 = x11207 + z11208 = x11208 + z11209 = x11209 + z11210 = x11210 + z11211 = x11211 + z11212 = x11212 + z11213 = x11213 + z11214 = x11214 + z11215 = x11215 + z11216 = x11216 + z11217 = x11217 + z11218 = x11218 + z11219 = x11219 + z11220 = x11220 + z11221 = x11221 + z11222 = x11222 + z11223 = x11223 + z11224 = x11224 + z11225 = x11225 + z11226 = x11226 + z11227 = x11227 + z11228 = x11228 + z11229 = x11229 + z11230 = x11230 + z11231 = x11231 + z11232 = x11232 + z11233 = x11233 + z11234 = x11234 + z11235 = x11235 + z11236 = x11236 + z11237 = x11237 + z11238 = x11238 + z11239 = x11239 + z11240 = x11240 + z11241 = x11241 + z11242 = x11242 + z11243 = x11243 + z11244 = x11244 + z11245 = x11245 + z11246 = x11246 + z11247 = x11247 + z11248 = x11248 + z11249 = x11249 + z11250 = x11250 + z11251 = x11251 + z11252 = x11252 + z11253 = x11253 + z11254 = x11254 + z11255 = x11255 + z11256 = x11256 + z11257 = x11257 + z11258 = x11258 + z11259 = x11259 + z11260 = x11260 + z11261 = x11261 + z11262 = x11262 + z11263 = x11263 + z11264 = x11264 + z11265 = x11265 + z11266 = x11266 + z11267 = x11267 + z11268 = x11268 + z11269 = x11269 + z11270 = x11270 + z11271 = x11271 + z11272 = x11272 + z11273 = x11273 + z11274 = x11274 + z11275 = x11275 + z11276 = x11276 + z11277 = x11277 + z11278 = x11278 + z11279 = x11279 + z11280 = x11280 + z11281 = x11281 + z11282 = x11282 + z11283 = x11283 + z11284 = x11284 + z11285 = x11285 + z11286 = x11286 + z11287 = x11287 + z11288 = x11288 + z11289 = x11289 + z11290 = x11290 + z11291 = x11291 + z11292 = x11292 + z11293 = x11293 + z11294 = x11294 + z11295 = x11295 + z11296 = x11296 + z11297 = x11297 + z11298 = x11298 + z11299 = x11299 + z11300 = x11300 + z11301 = x11301 + z11302 = x11302 + z11303 = x11303 + z11304 = x11304 + z11305 = x11305 + z11306 = x11306 + z11307 = x11307 + z11308 = x11308 + z11309 = x11309 + z11310 = x11310 + z11311 = x11311 + z11312 = x11312 + z11313 = x11313 + z11314 = x11314 + z11315 = x11315 + z11316 = x11316 + z11317 = x11317 + z11318 = x11318 + z11319 = x11319 + z11320 = x11320 + z11321 = x11321 + z11322 = x11322 + z11323 = x11323 + z11324 = x11324 + z11325 = x11325 + z11326 = x11326 + z11327 = x11327 + z11328 = x11328 + z11329 = x11329 + z11330 = x11330 + z11331 = x11331 + z11332 = x11332 + z11333 = x11333 + z11334 = x11334 + z11335 = x11335 + z11336 = x11336 + z11337 = x11337 + z11338 = x11338 + z11339 = x11339 + z11340 = x11340 + z11341 = x11341 + z11342 = x11342 + z11343 = x11343 + z11344 = x11344 + z11345 = x11345 + z11346 = x11346 + z11347 = x11347 + z11348 = x11348 + z11349 = x11349 + z11350 = x11350 + z11351 = x11351 + z11352 = x11352 + z11353 = x11353 + z11354 = x11354 + z11355 = x11355 + z11356 = x11356 + z11357 = x11357 + z11358 = x11358 + z11359 = x11359 + z11360 = x11360 + z11361 = x11361 + z11362 = x11362 + z11363 = x11363 + z11364 = x11364 + z11365 = x11365 + z11366 = x11366 + z11367 = x11367 + z11368 = x11368 + z11369 = x11369 + z11370 = x11370 + z11371 = x11371 + z11372 = x11372 + z11373 = x11373 + z11374 = x11374 + z11375 = x11375 + z11376 = x11376 + z11377 = x11377 + z11378 = x11378 + z11379 = x11379 + z11380 = x11380 + z11381 = x11381 + z11382 = x11382 + z11383 = x11383 + z11384 = x11384 + z11385 = x11385 + z11386 = x11386 + z11387 = x11387 + z11388 = x11388 + z11389 = x11389 + z11390 = x11390 + z11391 = x11391 + z11392 = x11392 + z11393 = x11393 + z11394 = x11394 + z11395 = x11395 + z11396 = x11396 + z11397 = x11397 + z11398 = x11398 + z11399 = x11399 + z11400 = x11400 + z11401 = x11401 + z11402 = x11402 + z11403 = x11403 + z11404 = x11404 + z11405 = x11405 + z11406 = x11406 + z11407 = x11407 + z11408 = x11408 + z11409 = x11409 + z11410 = x11410 + z11411 = x11411 + z11412 = x11412 + z11413 = x11413 + z11414 = x11414 + z11415 = x11415 + z11416 = x11416 + z11417 = x11417 + z11418 = x11418 + z11419 = x11419 + z11420 = x11420 + z11421 = x11421 + z11422 = x11422 + z11423 = x11423 + z11424 = x11424 + z11425 = x11425 + z11426 = x11426 + z11427 = x11427 + z11428 = x11428 + z11429 = x11429 + z11430 = x11430 + z11431 = x11431 + z11432 = x11432 + z11433 = x11433 + z11434 = x11434 + z11435 = x11435 + z11436 = x11436 + z11437 = x11437 + z11438 = x11438 + z11439 = x11439 + z11440 = x11440 + z11441 = x11441 + z11442 = x11442 + z11443 = x11443 + z11444 = x11444 + z11445 = x11445 + z11446 = x11446 + z11447 = x11447 + z11448 = x11448 + z11449 = x11449 + z11450 = x11450 + z11451 = x11451 + z11452 = x11452 + z11453 = x11453 + z11454 = x11454 + z11455 = x11455 + z11456 = x11456 + z11457 = x11457 + z11458 = x11458 + z11459 = x11459 + z11460 = x11460 + z11461 = x11461 + z11462 = x11462 + z11463 = x11463 + z11464 = x11464 + z11465 = x11465 + z11466 = x11466 + z11467 = x11467 + z11468 = x11468 + z11469 = x11469 + z11470 = x11470 + z11471 = x11471 + z11472 = x11472 + z11473 = x11473 + z11474 = x11474 + z11475 = x11475 + z11476 = x11476 + z11477 = x11477 + z11478 = x11478 + z11479 = x11479 + z11480 = x11480 + z11481 = x11481 + z11482 = x11482 + z11483 = x11483 + z11484 = x11484 + z11485 = x11485 + z11486 = x11486 + z11487 = x11487 + z11488 = x11488 + z11489 = x11489 + z11490 = x11490 + z11491 = x11491 + z11492 = x11492 + z11493 = x11493 + z11494 = x11494 + z11495 = x11495 + z11496 = x11496 + z11497 = x11497 + z11498 = x11498 + z11499 = x11499 + z11500 = x11500 + z11501 = x11501 + z11502 = x11502 + z11503 = x11503 + z11504 = x11504 + z11505 = x11505 + z11506 = x11506 + z11507 = x11507 + z11508 = x11508 + z11509 = x11509 + z11510 = x11510 + z11511 = x11511 + z11512 = x11512 + z11513 = x11513 + z11514 = x11514 + z11515 = x11515 + z11516 = x11516 + z11517 = x11517 + z11518 = x11518 + z11519 = x11519 + z11520 = x11520 + z11521 = x11521 + z11522 = x11522 + z11523 = x11523 + z11524 = x11524 + z11525 = x11525 + z11526 = x11526 + z11527 = x11527 + z11528 = x11528 + z11529 = x11529 + z11530 = x11530 + z11531 = x11531 + z11532 = x11532 + z11533 = x11533 + z11534 = x11534 + z11535 = x11535 + z11536 = x11536 + z11537 = x11537 + z11538 = x11538 + z11539 = x11539 + z11540 = x11540 + z11541 = x11541 + z11542 = x11542 + z11543 = x11543 + z11544 = x11544 + z11545 = x11545 + z11546 = x11546 + z11547 = x11547 + z11548 = x11548 + z11549 = x11549 + z11550 = x11550 + z11551 = x11551 + z11552 = x11552 + z11553 = x11553 + z11554 = x11554 + z11555 = x11555 + z11556 = x11556 + z11557 = x11557 + z11558 = x11558 + z11559 = x11559 + z11560 = x11560 + z11561 = x11561 + z11562 = x11562 + z11563 = x11563 + z11564 = x11564 + z11565 = x11565 + z11566 = x11566 + z11567 = x11567 + z11568 = x11568 + z11569 = x11569 + z11570 = x11570 + z11571 = x11571 + z11572 = x11572 + z11573 = x11573 + z11574 = x11574 + z11575 = x11575 + z11576 = x11576 + z11577 = x11577 + z11578 = x11578 + z11579 = x11579 + z11580 = x11580 + z11581 = x11581 + z11582 = x11582 + z11583 = x11583 + z11584 = x11584 + z11585 = x11585 + z11586 = x11586 + z11587 = x11587 + z11588 = x11588 + z11589 = x11589 + z11590 = x11590 + z11591 = x11591 + z11592 = x11592 + z11593 = x11593 + z11594 = x11594 + z11595 = x11595 + z11596 = x11596 + z11597 = x11597 + z11598 = x11598 + z11599 = x11599 + z11600 = x11600 + z11601 = x11601 + z11602 = x11602 + z11603 = x11603 + z11604 = x11604 + z11605 = x11605 + z11606 = x11606 + z11607 = x11607 + z11608 = x11608 + z11609 = x11609 + z11610 = x11610 + z11611 = x11611 + z11612 = x11612 + z11613 = x11613 + z11614 = x11614 + z11615 = x11615 + z11616 = x11616 + z11617 = x11617 + z11618 = x11618 + z11619 = x11619 + z11620 = x11620 + z11621 = x11621 + z11622 = x11622 + z11623 = x11623 + z11624 = x11624 + z11625 = x11625 + z11626 = x11626 + z11627 = x11627 + z11628 = x11628 + z11629 = x11629 + z11630 = x11630 + z11631 = x11631 + z11632 = x11632 + z11633 = x11633 + z11634 = x11634 + z11635 = x11635 + z11636 = x11636 + z11637 = x11637 + z11638 = x11638 + z11639 = x11639 + z11640 = x11640 + z11641 = x11641 + z11642 = x11642 + z11643 = x11643 + z11644 = x11644 + z11645 = x11645 + z11646 = x11646 + z11647 = x11647 + z11648 = x11648 + z11649 = x11649 + z11650 = x11650 + z11651 = x11651 + z11652 = x11652 + z11653 = x11653 + z11654 = x11654 + z11655 = x11655 + z11656 = x11656 + z11657 = x11657 + z11658 = x11658 + z11659 = x11659 + z11660 = x11660 + z11661 = x11661 + z11662 = x11662 + z11663 = x11663 + z11664 = x11664 + z11665 = x11665 + z11666 = x11666 + z11667 = x11667 + z11668 = x11668 + z11669 = x11669 + z11670 = x11670 + z11671 = x11671 + z11672 = x11672 + z11673 = x11673 + z11674 = x11674 + z11675 = x11675 + z11676 = x11676 + z11677 = x11677 + z11678 = x11678 + z11679 = x11679 + z11680 = x11680 + z11681 = x11681 + z11682 = x11682 + z11683 = x11683 + z11684 = x11684 + z11685 = x11685 + z11686 = x11686 + z11687 = x11687 + z11688 = x11688 + z11689 = x11689 + z11690 = x11690 + z11691 = x11691 + z11692 = x11692 + z11693 = x11693 + z11694 = x11694 + z11695 = x11695 + z11696 = x11696 + z11697 = x11697 + z11698 = x11698 + z11699 = x11699 + z11700 = x11700 + z11701 = x11701 + z11702 = x11702 + z11703 = x11703 + z11704 = x11704 + z11705 = x11705 + z11706 = x11706 + z11707 = x11707 + z11708 = x11708 + z11709 = x11709 + z11710 = x11710 + z11711 = x11711 + z11712 = x11712 + z11713 = x11713 + z11714 = x11714 + z11715 = x11715 + z11716 = x11716 + z11717 = x11717 + z11718 = x11718 + z11719 = x11719 + z11720 = x11720 + z11721 = x11721 + z11722 = x11722 + z11723 = x11723 + z11724 = x11724 + z11725 = x11725 + z11726 = x11726 + z11727 = x11727 + z11728 = x11728 + z11729 = x11729 + z11730 = x11730 + z11731 = x11731 + z11732 = x11732 + z11733 = x11733 + z11734 = x11734 + z11735 = x11735 + z11736 = x11736 + z11737 = x11737 + z11738 = x11738 + z11739 = x11739 + z11740 = x11740 + z11741 = x11741 + z11742 = x11742 + z11743 = x11743 + z11744 = x11744 + z11745 = x11745 + z11746 = x11746 + z11747 = x11747 + z11748 = x11748 + z11749 = x11749 + z11750 = x11750 + z11751 = x11751 + z11752 = x11752 + z11753 = x11753 + z11754 = x11754 + z11755 = x11755 + z11756 = x11756 + z11757 = x11757 + z11758 = x11758 + z11759 = x11759 + z11760 = x11760 + z11761 = x11761 + z11762 = x11762 + z11763 = x11763 + z11764 = x11764 + z11765 = x11765 + z11766 = x11766 + z11767 = x11767 + z11768 = x11768 + z11769 = x11769 + z11770 = x11770 + z11771 = x11771 + z11772 = x11772 + z11773 = x11773 + z11774 = x11774 + z11775 = x11775 + z11776 = x11776 + z11777 = x11777 + z11778 = x11778 + z11779 = x11779 + z11780 = x11780 + z11781 = x11781 + z11782 = x11782 + z11783 = x11783 + z11784 = x11784 + z11785 = x11785 + z11786 = x11786 + z11787 = x11787 + z11788 = x11788 + z11789 = x11789 + z11790 = x11790 + z11791 = x11791 + z11792 = x11792 + z11793 = x11793 + z11794 = x11794 + z11795 = x11795 + z11796 = x11796 + z11797 = x11797 + z11798 = x11798 + z11799 = x11799 + z11800 = x11800 + z11801 = x11801 + z11802 = x11802 + z11803 = x11803 + z11804 = x11804 + z11805 = x11805 + z11806 = x11806 + z11807 = x11807 + z11808 = x11808 + z11809 = x11809 + z11810 = x11810 + z11811 = x11811 + z11812 = x11812 + z11813 = x11813 + z11814 = x11814 + z11815 = x11815 + z11816 = x11816 + z11817 = x11817 + z11818 = x11818 + z11819 = x11819 + z11820 = x11820 + z11821 = x11821 + z11822 = x11822 + z11823 = x11823 + z11824 = x11824 + z11825 = x11825 + z11826 = x11826 + z11827 = x11827 + z11828 = x11828 + z11829 = x11829 + z11830 = x11830 + z11831 = x11831 + z11832 = x11832 + z11833 = x11833 + z11834 = x11834 + z11835 = x11835 + z11836 = x11836 + z11837 = x11837 + z11838 = x11838 + z11839 = x11839 + z11840 = x11840 + z11841 = x11841 + z11842 = x11842 + z11843 = x11843 + z11844 = x11844 + z11845 = x11845 + z11846 = x11846 + z11847 = x11847 + z11848 = x11848 + z11849 = x11849 + z11850 = x11850 + z11851 = x11851 + z11852 = x11852 + z11853 = x11853 + z11854 = x11854 + z11855 = x11855 + z11856 = x11856 + z11857 = x11857 + z11858 = x11858 + z11859 = x11859 + z11860 = x11860 + z11861 = x11861 + z11862 = x11862 + z11863 = x11863 + z11864 = x11864 + z11865 = x11865 + z11866 = x11866 + z11867 = x11867 + z11868 = x11868 + z11869 = x11869 + z11870 = x11870 + z11871 = x11871 + z11872 = x11872 + z11873 = x11873 + z11874 = x11874 + z11875 = x11875 + z11876 = x11876 + z11877 = x11877 + z11878 = x11878 + z11879 = x11879 + z11880 = x11880 + z11881 = x11881 + z11882 = x11882 + z11883 = x11883 + z11884 = x11884 + z11885 = x11885 + z11886 = x11886 + z11887 = x11887 + z11888 = x11888 + z11889 = x11889 + z11890 = x11890 + z11891 = x11891 + z11892 = x11892 + z11893 = x11893 + z11894 = x11894 + z11895 = x11895 + z11896 = x11896 + z11897 = x11897 + z11898 = x11898 + z11899 = x11899 + z11900 = x11900 + z11901 = x11901 + z11902 = x11902 + z11903 = x11903 + z11904 = x11904 + z11905 = x11905 + z11906 = x11906 + z11907 = x11907 + z11908 = x11908 + z11909 = x11909 + z11910 = x11910 + z11911 = x11911 + z11912 = x11912 + z11913 = x11913 + z11914 = x11914 + z11915 = x11915 + z11916 = x11916 + z11917 = x11917 + z11918 = x11918 + z11919 = x11919 + z11920 = x11920 + z11921 = x11921 + z11922 = x11922 + z11923 = x11923 + z11924 = x11924 + z11925 = x11925 + z11926 = x11926 + z11927 = x11927 + z11928 = x11928 + z11929 = x11929 + z11930 = x11930 + z11931 = x11931 + z11932 = x11932 + z11933 = x11933 + z11934 = x11934 + z11935 = x11935 + z11936 = x11936 + z11937 = x11937 + z11938 = x11938 + z11939 = x11939 + z11940 = x11940 + z11941 = x11941 + z11942 = x11942 + z11943 = x11943 + z11944 = x11944 + z11945 = x11945 + z11946 = x11946 + z11947 = x11947 + z11948 = x11948 + z11949 = x11949 + z11950 = x11950 + z11951 = x11951 + z11952 = x11952 + z11953 = x11953 + z11954 = x11954 + z11955 = x11955 + z11956 = x11956 + z11957 = x11957 + z11958 = x11958 + z11959 = x11959 + z11960 = x11960 + z11961 = x11961 + z11962 = x11962 + z11963 = x11963 + z11964 = x11964 + z11965 = x11965 + z11966 = x11966 + z11967 = x11967 + z11968 = x11968 + z11969 = x11969 + z11970 = x11970 + z11971 = x11971 + z11972 = x11972 + z11973 = x11973 + z11974 = x11974 + z11975 = x11975 + z11976 = x11976 + z11977 = x11977 + z11978 = x11978 + z11979 = x11979 + z11980 = x11980 + z11981 = x11981 + z11982 = x11982 + z11983 = x11983 + z11984 = x11984 + z11985 = x11985 + z11986 = x11986 + z11987 = x11987 + z11988 = x11988 + z11989 = x11989 + z11990 = x11990 + z11991 = x11991 + z11992 = x11992 + z11993 = x11993 + z11994 = x11994 + z11995 = x11995 + z11996 = x11996 + z11997 = x11997 + z11998 = x11998 + z11999 = x11999 + z12000 = x12000 + z12001 = x12001 + z12002 = x12002 + z12003 = x12003 + z12004 = x12004 + z12005 = x12005 + z12006 = x12006 + z12007 = x12007 + z12008 = x12008 + z12009 = x12009 + z12010 = x12010 + z12011 = x12011 + z12012 = x12012 + z12013 = x12013 + z12014 = x12014 + z12015 = x12015 + z12016 = x12016 + z12017 = x12017 + z12018 = x12018 + z12019 = x12019 + z12020 = x12020 + z12021 = x12021 + z12022 = x12022 + z12023 = x12023 + z12024 = x12024 + z12025 = x12025 + z12026 = x12026 + z12027 = x12027 + z12028 = x12028 + z12029 = x12029 + z12030 = x12030 + z12031 = x12031 + z12032 = x12032 + z12033 = x12033 + z12034 = x12034 + z12035 = x12035 + z12036 = x12036 + z12037 = x12037 + z12038 = x12038 + z12039 = x12039 + z12040 = x12040 + z12041 = x12041 + z12042 = x12042 + z12043 = x12043 + z12044 = x12044 + z12045 = x12045 + z12046 = x12046 + z12047 = x12047 + z12048 = x12048 + z12049 = x12049 + z12050 = x12050 + z12051 = x12051 + z12052 = x12052 + z12053 = x12053 + z12054 = x12054 + z12055 = x12055 + z12056 = x12056 + z12057 = x12057 + z12058 = x12058 + z12059 = x12059 + z12060 = x12060 + z12061 = x12061 + z12062 = x12062 + z12063 = x12063 + z12064 = x12064 + z12065 = x12065 + z12066 = x12066 + z12067 = x12067 + z12068 = x12068 + z12069 = x12069 + z12070 = x12070 + z12071 = x12071 + z12072 = x12072 + z12073 = x12073 + z12074 = x12074 + z12075 = x12075 + z12076 = x12076 + z12077 = x12077 + z12078 = x12078 + z12079 = x12079 + z12080 = x12080 + z12081 = x12081 + z12082 = x12082 + z12083 = x12083 + z12084 = x12084 + z12085 = x12085 + z12086 = x12086 + z12087 = x12087 + z12088 = x12088 + z12089 = x12089 + z12090 = x12090 + z12091 = x12091 + z12092 = x12092 + z12093 = x12093 + z12094 = x12094 + z12095 = x12095 + z12096 = x12096 + z12097 = x12097 + z12098 = x12098 + z12099 = x12099 + z12100 = x12100 + z12101 = x12101 + z12102 = x12102 + z12103 = x12103 + z12104 = x12104 + z12105 = x12105 + z12106 = x12106 + z12107 = x12107 + z12108 = x12108 + z12109 = x12109 + z12110 = x12110 + z12111 = x12111 + z12112 = x12112 + z12113 = x12113 + z12114 = x12114 + z12115 = x12115 + z12116 = x12116 + z12117 = x12117 + z12118 = x12118 + z12119 = x12119 + z12120 = x12120 + z12121 = x12121 + z12122 = x12122 + z12123 = x12123 + z12124 = x12124 + z12125 = x12125 + z12126 = x12126 + z12127 = x12127 + z12128 = x12128 + z12129 = x12129 + z12130 = x12130 + z12131 = x12131 + z12132 = x12132 + z12133 = x12133 + z12134 = x12134 + z12135 = x12135 + z12136 = x12136 + z12137 = x12137 + z12138 = x12138 + z12139 = x12139 + z12140 = x12140 + z12141 = x12141 + z12142 = x12142 + z12143 = x12143 + z12144 = x12144 + z12145 = x12145 + z12146 = x12146 + z12147 = x12147 + z12148 = x12148 + z12149 = x12149 + z12150 = x12150 + z12151 = x12151 + z12152 = x12152 + z12153 = x12153 + z12154 = x12154 + z12155 = x12155 + z12156 = x12156 + z12157 = x12157 + z12158 = x12158 + z12159 = x12159 + z12160 = x12160 + z12161 = x12161 + z12162 = x12162 + z12163 = x12163 + z12164 = x12164 + z12165 = x12165 + z12166 = x12166 + z12167 = x12167 + z12168 = x12168 + z12169 = x12169 + z12170 = x12170 + z12171 = x12171 + z12172 = x12172 + z12173 = x12173 + z12174 = x12174 + z12175 = x12175 + z12176 = x12176 + z12177 = x12177 + z12178 = x12178 + z12179 = x12179 + z12180 = x12180 + z12181 = x12181 + z12182 = x12182 + z12183 = x12183 + z12184 = x12184 + z12185 = x12185 + z12186 = x12186 + z12187 = x12187 + z12188 = x12188 + z12189 = x12189 + z12190 = x12190 + z12191 = x12191 + z12192 = x12192 + z12193 = x12193 + z12194 = x12194 + z12195 = x12195 + z12196 = x12196 + z12197 = x12197 + z12198 = x12198 + z12199 = x12199 + z12200 = x12200 + z12201 = x12201 + z12202 = x12202 + z12203 = x12203 + z12204 = x12204 + z12205 = x12205 + z12206 = x12206 + z12207 = x12207 + z12208 = x12208 + z12209 = x12209 + z12210 = x12210 + z12211 = x12211 + z12212 = x12212 + z12213 = x12213 + z12214 = x12214 + z12215 = x12215 + z12216 = x12216 + z12217 = x12217 + z12218 = x12218 + z12219 = x12219 + z12220 = x12220 + z12221 = x12221 + z12222 = x12222 + z12223 = x12223 + z12224 = x12224 + z12225 = x12225 + z12226 = x12226 + z12227 = x12227 + z12228 = x12228 + z12229 = x12229 + z12230 = x12230 + z12231 = x12231 + z12232 = x12232 + z12233 = x12233 + z12234 = x12234 + z12235 = x12235 + z12236 = x12236 + z12237 = x12237 + z12238 = x12238 + z12239 = x12239 + z12240 = x12240 + z12241 = x12241 + z12242 = x12242 + z12243 = x12243 + z12244 = x12244 + z12245 = x12245 + z12246 = x12246 + z12247 = x12247 + z12248 = x12248 + z12249 = x12249 + z12250 = x12250 + z12251 = x12251 + z12252 = x12252 + z12253 = x12253 + z12254 = x12254 + z12255 = x12255 + z12256 = x12256 + z12257 = x12257 + z12258 = x12258 + z12259 = x12259 + z12260 = x12260 + z12261 = x12261 + z12262 = x12262 + z12263 = x12263 + z12264 = x12264 + z12265 = x12265 + z12266 = x12266 + z12267 = x12267 + z12268 = x12268 + z12269 = x12269 + z12270 = x12270 + z12271 = x12271 + z12272 = x12272 + z12273 = x12273 + z12274 = x12274 + z12275 = x12275 + z12276 = x12276 + z12277 = x12277 + z12278 = x12278 + z12279 = x12279 + z12280 = x12280 + z12281 = x12281 + z12282 = x12282 + z12283 = x12283 + z12284 = x12284 + z12285 = x12285 + z12286 = x12286 + z12287 = x12287 + z12288 = x12288 + z12289 = x12289 + z12290 = x12290 + z12291 = x12291 + z12292 = x12292 + z12293 = x12293 + z12294 = x12294 + z12295 = x12295 + z12296 = x12296 + z12297 = x12297 + z12298 = x12298 + z12299 = x12299 + z12300 = x12300 + z12301 = x12301 + z12302 = x12302 + z12303 = x12303 + z12304 = x12304 + z12305 = x12305 + z12306 = x12306 + z12307 = x12307 + z12308 = x12308 + z12309 = x12309 + z12310 = x12310 + z12311 = x12311 + z12312 = x12312 + z12313 = x12313 + z12314 = x12314 + z12315 = x12315 + z12316 = x12316 + z12317 = x12317 + z12318 = x12318 + z12319 = x12319 + z12320 = x12320 + z12321 = x12321 + z12322 = x12322 + z12323 = x12323 + z12324 = x12324 + z12325 = x12325 + z12326 = x12326 + z12327 = x12327 + z12328 = x12328 + z12329 = x12329 + z12330 = x12330 + z12331 = x12331 + z12332 = x12332 + z12333 = x12333 + z12334 = x12334 + z12335 = x12335 + z12336 = x12336 + z12337 = x12337 + z12338 = x12338 + z12339 = x12339 + z12340 = x12340 + z12341 = x12341 + z12342 = x12342 + z12343 = x12343 + z12344 = x12344 + z12345 = x12345 + z12346 = x12346 + z12347 = x12347 + z12348 = x12348 + z12349 = x12349 + z12350 = x12350 + z12351 = x12351 + z12352 = x12352 + z12353 = x12353 + z12354 = x12354 + z12355 = x12355 + z12356 = x12356 + z12357 = x12357 + z12358 = x12358 + z12359 = x12359 + z12360 = x12360 + z12361 = x12361 + z12362 = x12362 + z12363 = x12363 + z12364 = x12364 + z12365 = x12365 + z12366 = x12366 + z12367 = x12367 + z12368 = x12368 + z12369 = x12369 + z12370 = x12370 + z12371 = x12371 + z12372 = x12372 + z12373 = x12373 + z12374 = x12374 + z12375 = x12375 + z12376 = x12376 + z12377 = x12377 + z12378 = x12378 + z12379 = x12379 + z12380 = x12380 + z12381 = x12381 + z12382 = x12382 + z12383 = x12383 + z12384 = x12384 + z12385 = x12385 + z12386 = x12386 + z12387 = x12387 + z12388 = x12388 + z12389 = x12389 + z12390 = x12390 + z12391 = x12391 + z12392 = x12392 + z12393 = x12393 + z12394 = x12394 + z12395 = x12395 + z12396 = x12396 + z12397 = x12397 + z12398 = x12398 + z12399 = x12399 + z12400 = x12400 + z12401 = x12401 + z12402 = x12402 + z12403 = x12403 + z12404 = x12404 + z12405 = x12405 + z12406 = x12406 + z12407 = x12407 + z12408 = x12408 + z12409 = x12409 + z12410 = x12410 + z12411 = x12411 + z12412 = x12412 + z12413 = x12413 + z12414 = x12414 + z12415 = x12415 + z12416 = x12416 + z12417 = x12417 + z12418 = x12418 + z12419 = x12419 + z12420 = x12420 + z12421 = x12421 + z12422 = x12422 + z12423 = x12423 + z12424 = x12424 + z12425 = x12425 + z12426 = x12426 + z12427 = x12427 + z12428 = x12428 + z12429 = x12429 + z12430 = x12430 + z12431 = x12431 + z12432 = x12432 + z12433 = x12433 + z12434 = x12434 + z12435 = x12435 + z12436 = x12436 + z12437 = x12437 + z12438 = x12438 + z12439 = x12439 + z12440 = x12440 + z12441 = x12441 + z12442 = x12442 + z12443 = x12443 + z12444 = x12444 + z12445 = x12445 + z12446 = x12446 + z12447 = x12447 + z12448 = x12448 + z12449 = x12449 + z12450 = x12450 + z12451 = x12451 + z12452 = x12452 + z12453 = x12453 + z12454 = x12454 + z12455 = x12455 + z12456 = x12456 + z12457 = x12457 + z12458 = x12458 + z12459 = x12459 + z12460 = x12460 + z12461 = x12461 + z12462 = x12462 + z12463 = x12463 + z12464 = x12464 + z12465 = x12465 + z12466 = x12466 + z12467 = x12467 + z12468 = x12468 + z12469 = x12469 + z12470 = x12470 + z12471 = x12471 + z12472 = x12472 + z12473 = x12473 + z12474 = x12474 + z12475 = x12475 + z12476 = x12476 + z12477 = x12477 + z12478 = x12478 + z12479 = x12479 + z12480 = x12480 + z12481 = x12481 + z12482 = x12482 + z12483 = x12483 + z12484 = x12484 + z12485 = x12485 + z12486 = x12486 + z12487 = x12487 + z12488 = x12488 + z12489 = x12489 + z12490 = x12490 + z12491 = x12491 + z12492 = x12492 + z12493 = x12493 + z12494 = x12494 + z12495 = x12495 + z12496 = x12496 + z12497 = x12497 + z12498 = x12498 + z12499 = x12499 + z12500 = x12500 + z12501 = x12501 + z12502 = x12502 + z12503 = x12503 + z12504 = x12504 + z12505 = x12505 + z12506 = x12506 + z12507 = x12507 + z12508 = x12508 + z12509 = x12509 + z12510 = x12510 + z12511 = x12511 + z12512 = x12512 + z12513 = x12513 + z12514 = x12514 + z12515 = x12515 + z12516 = x12516 + z12517 = x12517 + z12518 = x12518 + z12519 = x12519 + z12520 = x12520 + z12521 = x12521 + z12522 = x12522 + z12523 = x12523 + z12524 = x12524 + z12525 = x12525 + z12526 = x12526 + z12527 = x12527 + z12528 = x12528 + z12529 = x12529 + z12530 = x12530 + z12531 = x12531 + z12532 = x12532 + z12533 = x12533 + z12534 = x12534 + z12535 = x12535 + z12536 = x12536 + z12537 = x12537 + z12538 = x12538 + z12539 = x12539 + z12540 = x12540 + z12541 = x12541 + z12542 = x12542 + z12543 = x12543 + z12544 = x12544 + z12545 = x12545 + z12546 = x12546 + z12547 = x12547 + z12548 = x12548 + z12549 = x12549 + z12550 = x12550 + z12551 = x12551 + z12552 = x12552 + z12553 = x12553 + z12554 = x12554 + z12555 = x12555 + z12556 = x12556 + z12557 = x12557 + z12558 = x12558 + z12559 = x12559 + z12560 = x12560 + z12561 = x12561 + z12562 = x12562 + z12563 = x12563 + z12564 = x12564 + z12565 = x12565 + z12566 = x12566 + z12567 = x12567 + z12568 = x12568 + z12569 = x12569 + z12570 = x12570 + z12571 = x12571 + z12572 = x12572 + z12573 = x12573 + z12574 = x12574 + z12575 = x12575 + z12576 = x12576 + z12577 = x12577 + z12578 = x12578 + z12579 = x12579 + z12580 = x12580 + z12581 = x12581 + z12582 = x12582 + z12583 = x12583 + z12584 = x12584 + z12585 = x12585 + z12586 = x12586 + z12587 = x12587 + z12588 = x12588 + z12589 = x12589 + z12590 = x12590 + z12591 = x12591 + z12592 = x12592 + z12593 = x12593 + z12594 = x12594 + z12595 = x12595 + z12596 = x12596 + z12597 = x12597 + z12598 = x12598 + z12599 = x12599 + z12600 = x12600 + z12601 = x12601 + z12602 = x12602 + z12603 = x12603 + z12604 = x12604 + z12605 = x12605 + z12606 = x12606 + z12607 = x12607 + z12608 = x12608 + z12609 = x12609 + z12610 = x12610 + z12611 = x12611 + z12612 = x12612 + z12613 = x12613 + z12614 = x12614 + z12615 = x12615 + z12616 = x12616 + z12617 = x12617 + z12618 = x12618 + z12619 = x12619 + z12620 = x12620 + z12621 = x12621 + z12622 = x12622 + z12623 = x12623 + z12624 = x12624 + z12625 = x12625 + z12626 = x12626 + z12627 = x12627 + z12628 = x12628 + z12629 = x12629 + z12630 = x12630 + z12631 = x12631 + z12632 = x12632 + z12633 = x12633 + z12634 = x12634 + z12635 = x12635 + z12636 = x12636 + z12637 = x12637 + z12638 = x12638 + z12639 = x12639 + z12640 = x12640 + z12641 = x12641 + z12642 = x12642 + z12643 = x12643 + z12644 = x12644 + z12645 = x12645 + z12646 = x12646 + z12647 = x12647 + z12648 = x12648 + z12649 = x12649 + z12650 = x12650 + z12651 = x12651 + z12652 = x12652 + z12653 = x12653 + z12654 = x12654 + z12655 = x12655 + z12656 = x12656 + z12657 = x12657 + z12658 = x12658 + z12659 = x12659 + z12660 = x12660 + z12661 = x12661 + z12662 = x12662 + z12663 = x12663 + z12664 = x12664 + z12665 = x12665 + z12666 = x12666 + z12667 = x12667 + z12668 = x12668 + z12669 = x12669 + z12670 = x12670 + z12671 = x12671 + z12672 = x12672 + z12673 = x12673 + z12674 = x12674 + z12675 = x12675 + z12676 = x12676 + z12677 = x12677 + z12678 = x12678 + z12679 = x12679 + z12680 = x12680 + z12681 = x12681 + z12682 = x12682 + z12683 = x12683 + z12684 = x12684 + z12685 = x12685 + z12686 = x12686 + z12687 = x12687 + z12688 = x12688 + z12689 = x12689 + z12690 = x12690 + z12691 = x12691 + z12692 = x12692 + z12693 = x12693 + z12694 = x12694 + z12695 = x12695 + z12696 = x12696 + z12697 = x12697 + z12698 = x12698 + z12699 = x12699 + z12700 = x12700 + z12701 = x12701 + z12702 = x12702 + z12703 = x12703 + z12704 = x12704 + z12705 = x12705 + z12706 = x12706 + z12707 = x12707 + z12708 = x12708 + z12709 = x12709 + z12710 = x12710 + z12711 = x12711 + z12712 = x12712 + z12713 = x12713 + z12714 = x12714 + z12715 = x12715 + z12716 = x12716 + z12717 = x12717 + z12718 = x12718 + z12719 = x12719 + z12720 = x12720 + z12721 = x12721 + z12722 = x12722 + z12723 = x12723 + z12724 = x12724 + z12725 = x12725 + z12726 = x12726 + z12727 = x12727 + z12728 = x12728 + z12729 = x12729 + z12730 = x12730 + z12731 = x12731 + z12732 = x12732 + z12733 = x12733 + z12734 = x12734 + z12735 = x12735 + z12736 = x12736 + z12737 = x12737 + z12738 = x12738 + z12739 = x12739 + z12740 = x12740 + z12741 = x12741 + z12742 = x12742 + z12743 = x12743 + z12744 = x12744 + z12745 = x12745 + z12746 = x12746 + z12747 = x12747 + z12748 = x12748 + z12749 = x12749 + z12750 = x12750 + z12751 = x12751 + z12752 = x12752 + z12753 = x12753 + z12754 = x12754 + z12755 = x12755 + z12756 = x12756 + z12757 = x12757 + z12758 = x12758 + z12759 = x12759 + z12760 = x12760 + z12761 = x12761 + z12762 = x12762 + z12763 = x12763 + z12764 = x12764 + z12765 = x12765 + z12766 = x12766 + z12767 = x12767 + z12768 = x12768 + z12769 = x12769 + z12770 = x12770 + z12771 = x12771 + z12772 = x12772 + z12773 = x12773 + z12774 = x12774 + z12775 = x12775 + z12776 = x12776 + z12777 = x12777 + z12778 = x12778 + z12779 = x12779 + z12780 = x12780 + z12781 = x12781 + z12782 = x12782 + z12783 = x12783 + z12784 = x12784 + z12785 = x12785 + z12786 = x12786 + z12787 = x12787 + z12788 = x12788 + z12789 = x12789 + z12790 = x12790 + z12791 = x12791 + z12792 = x12792 + z12793 = x12793 + z12794 = x12794 + z12795 = x12795 + z12796 = x12796 + z12797 = x12797 + z12798 = x12798 + z12799 = x12799 + z12800 = x12800 + z12801 = x12801 + z12802 = x12802 + z12803 = x12803 + z12804 = x12804 + z12805 = x12805 + z12806 = x12806 + z12807 = x12807 + z12808 = x12808 + z12809 = x12809 + z12810 = x12810 + z12811 = x12811 + z12812 = x12812 + z12813 = x12813 + z12814 = x12814 + z12815 = x12815 + z12816 = x12816 + z12817 = x12817 + z12818 = x12818 + z12819 = x12819 + z12820 = x12820 + z12821 = x12821 + z12822 = x12822 + z12823 = x12823 + z12824 = x12824 + z12825 = x12825 + z12826 = x12826 + z12827 = x12827 + z12828 = x12828 + z12829 = x12829 + z12830 = x12830 + z12831 = x12831 + z12832 = x12832 + z12833 = x12833 + z12834 = x12834 + z12835 = x12835 + z12836 = x12836 + z12837 = x12837 + z12838 = x12838 + z12839 = x12839 + z12840 = x12840 + z12841 = x12841 + z12842 = x12842 + z12843 = x12843 + z12844 = x12844 + z12845 = x12845 + z12846 = x12846 + z12847 = x12847 + z12848 = x12848 + z12849 = x12849 + z12850 = x12850 + z12851 = x12851 + z12852 = x12852 + z12853 = x12853 + z12854 = x12854 + z12855 = x12855 + z12856 = x12856 + z12857 = x12857 + z12858 = x12858 + z12859 = x12859 + z12860 = x12860 + z12861 = x12861 + z12862 = x12862 + z12863 = x12863 + z12864 = x12864 + z12865 = x12865 + z12866 = x12866 + z12867 = x12867 + z12868 = x12868 + z12869 = x12869 + z12870 = x12870 + z12871 = x12871 + z12872 = x12872 + z12873 = x12873 + z12874 = x12874 + z12875 = x12875 + z12876 = x12876 + z12877 = x12877 + z12878 = x12878 + z12879 = x12879 + z12880 = x12880 + z12881 = x12881 + z12882 = x12882 + z12883 = x12883 + z12884 = x12884 + z12885 = x12885 + z12886 = x12886 + z12887 = x12887 + z12888 = x12888 + z12889 = x12889 + z12890 = x12890 + z12891 = x12891 + z12892 = x12892 + z12893 = x12893 + z12894 = x12894 + z12895 = x12895 + z12896 = x12896 + z12897 = x12897 + z12898 = x12898 + z12899 = x12899 + z12900 = x12900 + z12901 = x12901 + z12902 = x12902 + z12903 = x12903 + z12904 = x12904 + z12905 = x12905 + z12906 = x12906 + z12907 = x12907 + z12908 = x12908 + z12909 = x12909 + z12910 = x12910 + z12911 = x12911 + z12912 = x12912 + z12913 = x12913 + z12914 = x12914 + z12915 = x12915 + z12916 = x12916 + z12917 = x12917 + z12918 = x12918 + z12919 = x12919 + z12920 = x12920 + z12921 = x12921 + z12922 = x12922 + z12923 = x12923 + z12924 = x12924 + z12925 = x12925 + z12926 = x12926 + z12927 = x12927 + z12928 = x12928 + z12929 = x12929 + z12930 = x12930 + z12931 = x12931 + z12932 = x12932 + z12933 = x12933 + z12934 = x12934 + z12935 = x12935 + z12936 = x12936 + z12937 = x12937 + z12938 = x12938 + z12939 = x12939 + z12940 = x12940 + z12941 = x12941 + z12942 = x12942 + z12943 = x12943 + z12944 = x12944 + z12945 = x12945 + z12946 = x12946 + z12947 = x12947 + z12948 = x12948 + z12949 = x12949 + z12950 = x12950 + z12951 = x12951 + z12952 = x12952 + z12953 = x12953 + z12954 = x12954 + z12955 = x12955 + z12956 = x12956 + z12957 = x12957 + z12958 = x12958 + z12959 = x12959 + z12960 = x12960 + z12961 = x12961 + z12962 = x12962 + z12963 = x12963 + z12964 = x12964 + z12965 = x12965 + z12966 = x12966 + z12967 = x12967 + z12968 = x12968 + z12969 = x12969 + z12970 = x12970 + z12971 = x12971 + z12972 = x12972 + z12973 = x12973 + z12974 = x12974 + z12975 = x12975 + z12976 = x12976 + z12977 = x12977 + z12978 = x12978 + z12979 = x12979 + z12980 = x12980 + z12981 = x12981 + z12982 = x12982 + z12983 = x12983 + z12984 = x12984 + z12985 = x12985 + z12986 = x12986 + z12987 = x12987 + z12988 = x12988 + z12989 = x12989 + z12990 = x12990 + z12991 = x12991 + z12992 = x12992 + z12993 = x12993 + z12994 = x12994 + z12995 = x12995 + z12996 = x12996 + z12997 = x12997 + z12998 = x12998 + z12999 = x12999 + z13000 = x13000 + z13001 = x13001 + z13002 = x13002 + z13003 = x13003 + z13004 = x13004 + z13005 = x13005 + z13006 = x13006 + z13007 = x13007 + z13008 = x13008 + z13009 = x13009 + z13010 = x13010 + z13011 = x13011 + z13012 = x13012 + z13013 = x13013 + z13014 = x13014 + z13015 = x13015 + z13016 = x13016 + z13017 = x13017 + z13018 = x13018 + z13019 = x13019 + z13020 = x13020 + z13021 = x13021 + z13022 = x13022 + z13023 = x13023 + z13024 = x13024 + z13025 = x13025 + z13026 = x13026 + z13027 = x13027 + z13028 = x13028 + z13029 = x13029 + z13030 = x13030 + z13031 = x13031 + z13032 = x13032 + z13033 = x13033 + z13034 = x13034 + z13035 = x13035 + z13036 = x13036 + z13037 = x13037 + z13038 = x13038 + z13039 = x13039 + z13040 = x13040 + z13041 = x13041 + z13042 = x13042 + z13043 = x13043 + z13044 = x13044 + z13045 = x13045 + z13046 = x13046 + z13047 = x13047 + z13048 = x13048 + z13049 = x13049 + z13050 = x13050 + z13051 = x13051 + z13052 = x13052 + z13053 = x13053 + z13054 = x13054 + z13055 = x13055 + z13056 = x13056 + z13057 = x13057 + z13058 = x13058 + z13059 = x13059 + z13060 = x13060 + z13061 = x13061 + z13062 = x13062 + z13063 = x13063 + z13064 = x13064 + z13065 = x13065 + z13066 = x13066 + z13067 = x13067 + z13068 = x13068 + z13069 = x13069 + z13070 = x13070 + z13071 = x13071 + z13072 = x13072 + z13073 = x13073 + z13074 = x13074 + z13075 = x13075 + z13076 = x13076 + z13077 = x13077 + z13078 = x13078 + z13079 = x13079 + z13080 = x13080 + z13081 = x13081 + z13082 = x13082 + z13083 = x13083 + z13084 = x13084 + z13085 = x13085 + z13086 = x13086 + z13087 = x13087 + z13088 = x13088 + z13089 = x13089 + z13090 = x13090 + z13091 = x13091 + z13092 = x13092 + z13093 = x13093 + z13094 = x13094 + z13095 = x13095 + z13096 = x13096 + z13097 = x13097 + z13098 = x13098 + z13099 = x13099 + z13100 = x13100 + z13101 = x13101 + z13102 = x13102 + z13103 = x13103 + z13104 = x13104 + z13105 = x13105 + z13106 = x13106 + z13107 = x13107 + z13108 = x13108 + z13109 = x13109 + z13110 = x13110 + z13111 = x13111 + z13112 = x13112 + z13113 = x13113 + z13114 = x13114 + z13115 = x13115 + z13116 = x13116 + z13117 = x13117 + z13118 = x13118 + z13119 = x13119 + z13120 = x13120 + z13121 = x13121 + z13122 = x13122 + z13123 = x13123 + z13124 = x13124 + z13125 = x13125 + z13126 = x13126 + z13127 = x13127 + z13128 = x13128 + z13129 = x13129 + z13130 = x13130 + z13131 = x13131 + z13132 = x13132 + z13133 = x13133 + z13134 = x13134 + z13135 = x13135 + z13136 = x13136 + z13137 = x13137 + z13138 = x13138 + z13139 = x13139 + z13140 = x13140 + z13141 = x13141 + z13142 = x13142 + z13143 = x13143 + z13144 = x13144 + z13145 = x13145 + z13146 = x13146 + z13147 = x13147 + z13148 = x13148 + z13149 = x13149 + z13150 = x13150 + z13151 = x13151 + z13152 = x13152 + z13153 = x13153 + z13154 = x13154 + z13155 = x13155 + z13156 = x13156 + z13157 = x13157 + z13158 = x13158 + z13159 = x13159 + z13160 = x13160 + z13161 = x13161 + z13162 = x13162 + z13163 = x13163 + z13164 = x13164 + z13165 = x13165 + z13166 = x13166 + z13167 = x13167 + z13168 = x13168 + z13169 = x13169 + z13170 = x13170 + z13171 = x13171 + z13172 = x13172 + z13173 = x13173 + z13174 = x13174 + z13175 = x13175 + z13176 = x13176 + z13177 = x13177 + z13178 = x13178 + z13179 = x13179 + z13180 = x13180 + z13181 = x13181 + z13182 = x13182 + z13183 = x13183 + z13184 = x13184 + z13185 = x13185 + z13186 = x13186 + z13187 = x13187 + z13188 = x13188 + z13189 = x13189 + z13190 = x13190 + z13191 = x13191 + z13192 = x13192 + z13193 = x13193 + z13194 = x13194 + z13195 = x13195 + z13196 = x13196 + z13197 = x13197 + z13198 = x13198 + z13199 = x13199 + z13200 = x13200 + z13201 = x13201 + z13202 = x13202 + z13203 = x13203 + z13204 = x13204 + z13205 = x13205 + z13206 = x13206 + z13207 = x13207 + z13208 = x13208 + z13209 = x13209 + z13210 = x13210 + z13211 = x13211 + z13212 = x13212 + z13213 = x13213 + z13214 = x13214 + z13215 = x13215 + z13216 = x13216 + z13217 = x13217 + z13218 = x13218 + z13219 = x13219 + z13220 = x13220 + z13221 = x13221 + z13222 = x13222 + z13223 = x13223 + z13224 = x13224 + z13225 = x13225 + z13226 = x13226 + z13227 = x13227 + z13228 = x13228 + z13229 = x13229 + z13230 = x13230 + z13231 = x13231 + z13232 = x13232 + z13233 = x13233 + z13234 = x13234 + z13235 = x13235 + z13236 = x13236 + z13237 = x13237 + z13238 = x13238 + z13239 = x13239 + z13240 = x13240 + z13241 = x13241 + z13242 = x13242 + z13243 = x13243 + z13244 = x13244 + z13245 = x13245 + z13246 = x13246 + z13247 = x13247 + z13248 = x13248 + z13249 = x13249 + z13250 = x13250 + z13251 = x13251 + z13252 = x13252 + z13253 = x13253 + z13254 = x13254 + z13255 = x13255 + z13256 = x13256 + z13257 = x13257 + z13258 = x13258 + z13259 = x13259 + z13260 = x13260 + z13261 = x13261 + z13262 = x13262 + z13263 = x13263 + z13264 = x13264 + z13265 = x13265 + z13266 = x13266 + z13267 = x13267 + z13268 = x13268 + z13269 = x13269 + z13270 = x13270 + z13271 = x13271 + z13272 = x13272 + z13273 = x13273 + z13274 = x13274 + z13275 = x13275 + z13276 = x13276 + z13277 = x13277 + z13278 = x13278 + z13279 = x13279 + z13280 = x13280 + z13281 = x13281 + z13282 = x13282 + z13283 = x13283 + z13284 = x13284 + z13285 = x13285 + z13286 = x13286 + z13287 = x13287 + z13288 = x13288 + z13289 = x13289 + z13290 = x13290 + z13291 = x13291 + z13292 = x13292 + z13293 = x13293 + z13294 = x13294 + z13295 = x13295 + z13296 = x13296 + z13297 = x13297 + z13298 = x13298 + z13299 = x13299 + z13300 = x13300 + z13301 = x13301 + z13302 = x13302 + z13303 = x13303 + z13304 = x13304 + z13305 = x13305 + z13306 = x13306 + z13307 = x13307 + z13308 = x13308 + z13309 = x13309 + z13310 = x13310 + z13311 = x13311 + z13312 = x13312 + z13313 = x13313 + z13314 = x13314 + z13315 = x13315 + z13316 = x13316 + z13317 = x13317 + z13318 = x13318 + z13319 = x13319 + z13320 = x13320 + z13321 = x13321 + z13322 = x13322 + z13323 = x13323 + z13324 = x13324 + z13325 = x13325 + z13326 = x13326 + z13327 = x13327 + z13328 = x13328 + z13329 = x13329 + z13330 = x13330 + z13331 = x13331 + z13332 = x13332 + z13333 = x13333 + z13334 = x13334 + z13335 = x13335 + z13336 = x13336 + z13337 = x13337 + z13338 = x13338 + z13339 = x13339 + z13340 = x13340 + z13341 = x13341 + z13342 = x13342 + z13343 = x13343 + z13344 = x13344 + z13345 = x13345 + z13346 = x13346 + z13347 = x13347 + z13348 = x13348 + z13349 = x13349 + z13350 = x13350 + z13351 = x13351 + z13352 = x13352 + z13353 = x13353 + z13354 = x13354 + z13355 = x13355 + z13356 = x13356 + z13357 = x13357 + z13358 = x13358 + z13359 = x13359 + z13360 = x13360 + z13361 = x13361 + z13362 = x13362 + z13363 = x13363 + z13364 = x13364 + z13365 = x13365 + z13366 = x13366 + z13367 = x13367 + z13368 = x13368 + z13369 = x13369 + z13370 = x13370 + z13371 = x13371 + z13372 = x13372 + z13373 = x13373 + z13374 = x13374 + z13375 = x13375 + z13376 = x13376 + z13377 = x13377 + z13378 = x13378 + z13379 = x13379 + z13380 = x13380 + z13381 = x13381 + z13382 = x13382 + z13383 = x13383 + z13384 = x13384 + z13385 = x13385 + z13386 = x13386 + z13387 = x13387 + z13388 = x13388 + z13389 = x13389 + z13390 = x13390 + z13391 = x13391 + z13392 = x13392 + z13393 = x13393 + z13394 = x13394 + z13395 = x13395 + z13396 = x13396 + z13397 = x13397 + z13398 = x13398 + z13399 = x13399 + z13400 = x13400 + z13401 = x13401 + z13402 = x13402 + z13403 = x13403 + z13404 = x13404 + z13405 = x13405 + z13406 = x13406 + z13407 = x13407 + z13408 = x13408 + z13409 = x13409 + z13410 = x13410 + z13411 = x13411 + z13412 = x13412 + z13413 = x13413 + z13414 = x13414 + z13415 = x13415 + z13416 = x13416 + z13417 = x13417 + z13418 = x13418 + z13419 = x13419 + z13420 = x13420 + z13421 = x13421 + z13422 = x13422 + z13423 = x13423 + z13424 = x13424 + z13425 = x13425 + z13426 = x13426 + z13427 = x13427 + z13428 = x13428 + z13429 = x13429 + z13430 = x13430 + z13431 = x13431 + z13432 = x13432 + z13433 = x13433 + z13434 = x13434 + z13435 = x13435 + z13436 = x13436 + z13437 = x13437 + z13438 = x13438 + z13439 = x13439 + z13440 = x13440 + z13441 = x13441 + z13442 = x13442 + z13443 = x13443 + z13444 = x13444 + z13445 = x13445 + z13446 = x13446 + z13447 = x13447 + z13448 = x13448 + z13449 = x13449 + z13450 = x13450 + z13451 = x13451 + z13452 = x13452 + z13453 = x13453 + z13454 = x13454 + z13455 = x13455 + z13456 = x13456 + z13457 = x13457 + z13458 = x13458 + z13459 = x13459 + z13460 = x13460 + z13461 = x13461 + z13462 = x13462 + z13463 = x13463 + z13464 = x13464 + z13465 = x13465 + z13466 = x13466 + z13467 = x13467 + z13468 = x13468 + z13469 = x13469 + z13470 = x13470 + z13471 = x13471 + z13472 = x13472 + z13473 = x13473 + z13474 = x13474 + z13475 = x13475 + z13476 = x13476 + z13477 = x13477 + z13478 = x13478 + z13479 = x13479 + z13480 = x13480 + z13481 = x13481 + z13482 = x13482 + z13483 = x13483 + z13484 = x13484 + z13485 = x13485 + z13486 = x13486 + z13487 = x13487 + z13488 = x13488 + z13489 = x13489 + z13490 = x13490 + z13491 = x13491 + z13492 = x13492 + z13493 = x13493 + z13494 = x13494 + z13495 = x13495 + z13496 = x13496 + z13497 = x13497 + z13498 = x13498 + z13499 = x13499 + z13500 = x13500 + z13501 = x13501 + z13502 = x13502 + z13503 = x13503 + z13504 = x13504 + z13505 = x13505 + z13506 = x13506 + z13507 = x13507 + z13508 = x13508 + z13509 = x13509 + z13510 = x13510 + z13511 = x13511 + z13512 = x13512 + z13513 = x13513 + z13514 = x13514 + z13515 = x13515 + z13516 = x13516 + z13517 = x13517 + z13518 = x13518 + z13519 = x13519 + z13520 = x13520 + z13521 = x13521 + z13522 = x13522 + z13523 = x13523 + z13524 = x13524 + z13525 = x13525 + z13526 = x13526 + z13527 = x13527 + z13528 = x13528 + z13529 = x13529 + z13530 = x13530 + z13531 = x13531 + z13532 = x13532 + z13533 = x13533 + z13534 = x13534 + z13535 = x13535 + z13536 = x13536 + z13537 = x13537 + z13538 = x13538 + z13539 = x13539 + z13540 = x13540 + z13541 = x13541 + z13542 = x13542 + z13543 = x13543 + z13544 = x13544 + z13545 = x13545 + z13546 = x13546 + z13547 = x13547 + z13548 = x13548 + z13549 = x13549 + z13550 = x13550 + z13551 = x13551 + z13552 = x13552 + z13553 = x13553 + z13554 = x13554 + z13555 = x13555 + z13556 = x13556 + z13557 = x13557 + z13558 = x13558 + z13559 = x13559 + z13560 = x13560 + z13561 = x13561 + z13562 = x13562 + z13563 = x13563 + z13564 = x13564 + z13565 = x13565 + z13566 = x13566 + z13567 = x13567 + z13568 = x13568 + z13569 = x13569 + z13570 = x13570 + z13571 = x13571 + z13572 = x13572 + z13573 = x13573 + z13574 = x13574 + z13575 = x13575 + z13576 = x13576 + z13577 = x13577 + z13578 = x13578 + z13579 = x13579 + z13580 = x13580 + z13581 = x13581 + z13582 = x13582 + z13583 = x13583 + z13584 = x13584 + z13585 = x13585 + z13586 = x13586 + z13587 = x13587 + z13588 = x13588 + z13589 = x13589 + z13590 = x13590 + z13591 = x13591 + z13592 = x13592 + z13593 = x13593 + z13594 = x13594 + z13595 = x13595 + z13596 = x13596 + z13597 = x13597 + z13598 = x13598 + z13599 = x13599 + z13600 = x13600 + z13601 = x13601 + z13602 = x13602 + z13603 = x13603 + z13604 = x13604 + z13605 = x13605 + z13606 = x13606 + z13607 = x13607 + z13608 = x13608 + z13609 = x13609 + z13610 = x13610 + z13611 = x13611 + z13612 = x13612 + z13613 = x13613 + z13614 = x13614 + z13615 = x13615 + z13616 = x13616 + z13617 = x13617 + z13618 = x13618 + z13619 = x13619 + z13620 = x13620 + z13621 = x13621 + z13622 = x13622 + z13623 = x13623 + z13624 = x13624 + z13625 = x13625 + z13626 = x13626 + z13627 = x13627 + z13628 = x13628 + z13629 = x13629 + z13630 = x13630 + z13631 = x13631 + z13632 = x13632 + z13633 = x13633 + z13634 = x13634 + z13635 = x13635 + z13636 = x13636 + z13637 = x13637 + z13638 = x13638 + z13639 = x13639 + z13640 = x13640 + z13641 = x13641 + z13642 = x13642 + z13643 = x13643 + z13644 = x13644 + z13645 = x13645 + z13646 = x13646 + z13647 = x13647 + z13648 = x13648 + z13649 = x13649 + z13650 = x13650 + z13651 = x13651 + z13652 = x13652 + z13653 = x13653 + z13654 = x13654 + z13655 = x13655 + z13656 = x13656 + z13657 = x13657 + z13658 = x13658 + z13659 = x13659 + z13660 = x13660 + z13661 = x13661 + z13662 = x13662 + z13663 = x13663 + z13664 = x13664 + z13665 = x13665 + z13666 = x13666 + z13667 = x13667 + z13668 = x13668 + z13669 = x13669 + z13670 = x13670 + z13671 = x13671 + z13672 = x13672 + z13673 = x13673 + z13674 = x13674 + z13675 = x13675 + z13676 = x13676 + z13677 = x13677 + z13678 = x13678 + z13679 = x13679 + z13680 = x13680 + z13681 = x13681 + z13682 = x13682 + z13683 = x13683 + z13684 = x13684 + z13685 = x13685 + z13686 = x13686 + z13687 = x13687 + z13688 = x13688 + z13689 = x13689 + z13690 = x13690 + z13691 = x13691 + z13692 = x13692 + z13693 = x13693 + z13694 = x13694 + z13695 = x13695 + z13696 = x13696 + z13697 = x13697 + z13698 = x13698 + z13699 = x13699 + z13700 = x13700 + z13701 = x13701 + z13702 = x13702 + z13703 = x13703 + z13704 = x13704 + z13705 = x13705 + z13706 = x13706 + z13707 = x13707 + z13708 = x13708 + z13709 = x13709 + z13710 = x13710 + z13711 = x13711 + z13712 = x13712 + z13713 = x13713 + z13714 = x13714 + z13715 = x13715 + z13716 = x13716 + z13717 = x13717 + z13718 = x13718 + z13719 = x13719 + z13720 = x13720 + z13721 = x13721 + z13722 = x13722 + z13723 = x13723 + z13724 = x13724 + z13725 = x13725 + z13726 = x13726 + z13727 = x13727 + z13728 = x13728 + z13729 = x13729 + z13730 = x13730 + z13731 = x13731 + z13732 = x13732 + z13733 = x13733 + z13734 = x13734 + z13735 = x13735 + z13736 = x13736 + z13737 = x13737 + z13738 = x13738 + z13739 = x13739 + z13740 = x13740 + z13741 = x13741 + z13742 = x13742 + z13743 = x13743 + z13744 = x13744 + z13745 = x13745 + z13746 = x13746 + z13747 = x13747 + z13748 = x13748 + z13749 = x13749 + z13750 = x13750 + z13751 = x13751 + z13752 = x13752 + z13753 = x13753 + z13754 = x13754 + z13755 = x13755 + z13756 = x13756 + z13757 = x13757 + z13758 = x13758 + z13759 = x13759 + z13760 = x13760 + z13761 = x13761 + z13762 = x13762 + z13763 = x13763 + z13764 = x13764 + z13765 = x13765 + z13766 = x13766 + z13767 = x13767 + z13768 = x13768 + z13769 = x13769 + z13770 = x13770 + z13771 = x13771 + z13772 = x13772 + z13773 = x13773 + z13774 = x13774 + z13775 = x13775 + z13776 = x13776 + z13777 = x13777 + z13778 = x13778 + z13779 = x13779 + z13780 = x13780 + z13781 = x13781 + z13782 = x13782 + z13783 = x13783 + z13784 = x13784 + z13785 = x13785 + z13786 = x13786 + z13787 = x13787 + z13788 = x13788 + z13789 = x13789 + z13790 = x13790 + z13791 = x13791 + z13792 = x13792 + z13793 = x13793 + z13794 = x13794 + z13795 = x13795 + z13796 = x13796 + z13797 = x13797 + z13798 = x13798 + z13799 = x13799 + z13800 = x13800 + z13801 = x13801 + z13802 = x13802 + z13803 = x13803 + z13804 = x13804 + z13805 = x13805 + z13806 = x13806 + z13807 = x13807 + z13808 = x13808 + z13809 = x13809 + z13810 = x13810 + z13811 = x13811 + z13812 = x13812 + z13813 = x13813 + z13814 = x13814 + z13815 = x13815 + z13816 = x13816 + z13817 = x13817 + z13818 = x13818 + z13819 = x13819 + z13820 = x13820 + z13821 = x13821 + z13822 = x13822 + z13823 = x13823 + z13824 = x13824 + z13825 = x13825 + z13826 = x13826 + z13827 = x13827 + z13828 = x13828 + z13829 = x13829 + z13830 = x13830 + z13831 = x13831 + z13832 = x13832 + z13833 = x13833 + z13834 = x13834 + z13835 = x13835 + z13836 = x13836 + z13837 = x13837 + z13838 = x13838 + z13839 = x13839 + z13840 = x13840 + z13841 = x13841 + z13842 = x13842 + z13843 = x13843 + z13844 = x13844 + z13845 = x13845 + z13846 = x13846 + z13847 = x13847 + z13848 = x13848 + z13849 = x13849 + z13850 = x13850 + z13851 = x13851 + z13852 = x13852 + z13853 = x13853 + z13854 = x13854 + z13855 = x13855 + z13856 = x13856 + z13857 = x13857 + z13858 = x13858 + z13859 = x13859 + z13860 = x13860 + z13861 = x13861 + z13862 = x13862 + z13863 = x13863 + z13864 = x13864 + z13865 = x13865 + z13866 = x13866 + z13867 = x13867 + z13868 = x13868 + z13869 = x13869 + z13870 = x13870 + z13871 = x13871 + z13872 = x13872 + z13873 = x13873 + z13874 = x13874 + z13875 = x13875 + z13876 = x13876 + z13877 = x13877 + z13878 = x13878 + z13879 = x13879 + z13880 = x13880 + z13881 = x13881 + z13882 = x13882 + z13883 = x13883 + z13884 = x13884 + z13885 = x13885 + z13886 = x13886 + z13887 = x13887 + z13888 = x13888 + z13889 = x13889 + z13890 = x13890 + z13891 = x13891 + z13892 = x13892 + z13893 = x13893 + z13894 = x13894 + z13895 = x13895 + z13896 = x13896 + z13897 = x13897 + z13898 = x13898 + z13899 = x13899 + z13900 = x13900 + z13901 = x13901 + z13902 = x13902 + z13903 = x13903 + z13904 = x13904 + z13905 = x13905 + z13906 = x13906 + z13907 = x13907 + z13908 = x13908 + z13909 = x13909 + z13910 = x13910 + z13911 = x13911 + z13912 = x13912 + z13913 = x13913 + z13914 = x13914 + z13915 = x13915 + z13916 = x13916 + z13917 = x13917 + z13918 = x13918 + z13919 = x13919 + z13920 = x13920 + z13921 = x13921 + z13922 = x13922 + z13923 = x13923 + z13924 = x13924 + z13925 = x13925 + z13926 = x13926 + z13927 = x13927 + z13928 = x13928 + z13929 = x13929 + z13930 = x13930 + z13931 = x13931 + z13932 = x13932 + z13933 = x13933 + z13934 = x13934 + z13935 = x13935 + z13936 = x13936 + z13937 = x13937 + z13938 = x13938 + z13939 = x13939 + z13940 = x13940 + z13941 = x13941 + z13942 = x13942 + z13943 = x13943 + z13944 = x13944 + z13945 = x13945 + z13946 = x13946 + z13947 = x13947 + z13948 = x13948 + z13949 = x13949 + z13950 = x13950 + z13951 = x13951 + z13952 = x13952 + z13953 = x13953 + z13954 = x13954 + z13955 = x13955 + z13956 = x13956 + z13957 = x13957 + z13958 = x13958 + z13959 = x13959 + z13960 = x13960 + z13961 = x13961 + z13962 = x13962 + z13963 = x13963 + z13964 = x13964 + z13965 = x13965 + z13966 = x13966 + z13967 = x13967 + z13968 = x13968 + z13969 = x13969 + z13970 = x13970 + z13971 = x13971 + z13972 = x13972 + z13973 = x13973 + z13974 = x13974 + z13975 = x13975 + z13976 = x13976 + z13977 = x13977 + z13978 = x13978 + z13979 = x13979 + z13980 = x13980 + z13981 = x13981 + z13982 = x13982 + z13983 = x13983 + z13984 = x13984 + z13985 = x13985 + z13986 = x13986 + z13987 = x13987 + z13988 = x13988 + z13989 = x13989 + z13990 = x13990 + z13991 = x13991 + z13992 = x13992 + z13993 = x13993 + z13994 = x13994 + z13995 = x13995 + z13996 = x13996 + z13997 = x13997 + z13998 = x13998 + z13999 = x13999 + z14000 = x14000 + z14001 = x14001 + z14002 = x14002 + z14003 = x14003 + z14004 = x14004 + z14005 = x14005 + z14006 = x14006 + z14007 = x14007 + z14008 = x14008 + z14009 = x14009 + z14010 = x14010 + z14011 = x14011 + z14012 = x14012 + z14013 = x14013 + z14014 = x14014 + z14015 = x14015 + z14016 = x14016 + z14017 = x14017 + z14018 = x14018 + z14019 = x14019 + z14020 = x14020 + z14021 = x14021 + z14022 = x14022 + z14023 = x14023 + z14024 = x14024 + z14025 = x14025 + z14026 = x14026 + z14027 = x14027 + z14028 = x14028 + z14029 = x14029 + z14030 = x14030 + z14031 = x14031 + z14032 = x14032 + z14033 = x14033 + z14034 = x14034 + z14035 = x14035 + z14036 = x14036 + z14037 = x14037 + z14038 = x14038 + z14039 = x14039 + z14040 = x14040 + z14041 = x14041 + z14042 = x14042 + z14043 = x14043 + z14044 = x14044 + z14045 = x14045 + z14046 = x14046 + z14047 = x14047 + z14048 = x14048 + z14049 = x14049 + z14050 = x14050 + z14051 = x14051 + z14052 = x14052 + z14053 = x14053 + z14054 = x14054 + z14055 = x14055 + z14056 = x14056 + z14057 = x14057 + z14058 = x14058 + z14059 = x14059 + z14060 = x14060 + z14061 = x14061 + z14062 = x14062 + z14063 = x14063 + z14064 = x14064 + z14065 = x14065 + z14066 = x14066 + z14067 = x14067 + z14068 = x14068 + z14069 = x14069 + z14070 = x14070 + z14071 = x14071 + z14072 = x14072 + z14073 = x14073 + z14074 = x14074 + z14075 = x14075 + z14076 = x14076 + z14077 = x14077 + z14078 = x14078 + z14079 = x14079 + z14080 = x14080 + z14081 = x14081 + z14082 = x14082 + z14083 = x14083 + z14084 = x14084 + z14085 = x14085 + z14086 = x14086 + z14087 = x14087 + z14088 = x14088 + z14089 = x14089 + z14090 = x14090 + z14091 = x14091 + z14092 = x14092 + z14093 = x14093 + z14094 = x14094 + z14095 = x14095 + z14096 = x14096 + z14097 = x14097 + z14098 = x14098 + z14099 = x14099 + z14100 = x14100 + z14101 = x14101 + z14102 = x14102 + z14103 = x14103 + z14104 = x14104 + z14105 = x14105 + z14106 = x14106 + z14107 = x14107 + z14108 = x14108 + z14109 = x14109 + z14110 = x14110 + z14111 = x14111 + z14112 = x14112 + z14113 = x14113 + z14114 = x14114 + z14115 = x14115 + z14116 = x14116 + z14117 = x14117 + z14118 = x14118 + z14119 = x14119 + z14120 = x14120 + z14121 = x14121 + z14122 = x14122 + z14123 = x14123 + z14124 = x14124 + z14125 = x14125 + z14126 = x14126 + z14127 = x14127 + z14128 = x14128 + z14129 = x14129 + z14130 = x14130 + z14131 = x14131 + z14132 = x14132 + z14133 = x14133 + z14134 = x14134 + z14135 = x14135 + z14136 = x14136 + z14137 = x14137 + z14138 = x14138 + z14139 = x14139 + z14140 = x14140 + z14141 = x14141 + z14142 = x14142 + z14143 = x14143 + z14144 = x14144 + z14145 = x14145 + z14146 = x14146 + z14147 = x14147 + z14148 = x14148 + z14149 = x14149 + z14150 = x14150 + z14151 = x14151 + z14152 = x14152 + z14153 = x14153 + z14154 = x14154 + z14155 = x14155 + z14156 = x14156 + z14157 = x14157 + z14158 = x14158 + z14159 = x14159 + z14160 = x14160 + z14161 = x14161 + z14162 = x14162 + z14163 = x14163 + z14164 = x14164 + z14165 = x14165 + z14166 = x14166 + z14167 = x14167 + z14168 = x14168 + z14169 = x14169 + z14170 = x14170 + z14171 = x14171 + z14172 = x14172 + z14173 = x14173 + z14174 = x14174 + z14175 = x14175 + z14176 = x14176 + z14177 = x14177 + z14178 = x14178 + z14179 = x14179 + z14180 = x14180 + z14181 = x14181 + z14182 = x14182 + z14183 = x14183 + z14184 = x14184 + z14185 = x14185 + z14186 = x14186 + z14187 = x14187 + z14188 = x14188 + z14189 = x14189 + z14190 = x14190 + z14191 = x14191 + z14192 = x14192 + z14193 = x14193 + z14194 = x14194 + z14195 = x14195 + z14196 = x14196 + z14197 = x14197 + z14198 = x14198 + z14199 = x14199 + z14200 = x14200 + z14201 = x14201 + z14202 = x14202 + z14203 = x14203 + z14204 = x14204 + z14205 = x14205 + z14206 = x14206 + z14207 = x14207 + z14208 = x14208 + z14209 = x14209 + z14210 = x14210 + z14211 = x14211 + z14212 = x14212 + z14213 = x14213 + z14214 = x14214 + z14215 = x14215 + z14216 = x14216 + z14217 = x14217 + z14218 = x14218 + z14219 = x14219 + z14220 = x14220 + z14221 = x14221 + z14222 = x14222 + z14223 = x14223 + z14224 = x14224 + z14225 = x14225 + z14226 = x14226 + z14227 = x14227 + z14228 = x14228 + z14229 = x14229 + z14230 = x14230 + z14231 = x14231 + z14232 = x14232 + z14233 = x14233 + z14234 = x14234 + z14235 = x14235 + z14236 = x14236 + z14237 = x14237 + z14238 = x14238 + z14239 = x14239 + z14240 = x14240 + z14241 = x14241 + z14242 = x14242 + z14243 = x14243 + z14244 = x14244 + z14245 = x14245 + z14246 = x14246 + z14247 = x14247 + z14248 = x14248 + z14249 = x14249 + z14250 = x14250 + z14251 = x14251 + z14252 = x14252 + z14253 = x14253 + z14254 = x14254 + z14255 = x14255 + z14256 = x14256 + z14257 = x14257 + z14258 = x14258 + z14259 = x14259 + z14260 = x14260 + z14261 = x14261 + z14262 = x14262 + z14263 = x14263 + z14264 = x14264 + z14265 = x14265 + z14266 = x14266 + z14267 = x14267 + z14268 = x14268 + z14269 = x14269 + z14270 = x14270 + z14271 = x14271 + z14272 = x14272 + z14273 = x14273 + z14274 = x14274 + z14275 = x14275 + z14276 = x14276 + z14277 = x14277 + z14278 = x14278 + z14279 = x14279 + z14280 = x14280 + z14281 = x14281 + z14282 = x14282 + z14283 = x14283 + z14284 = x14284 + z14285 = x14285 + z14286 = x14286 + z14287 = x14287 + z14288 = x14288 + z14289 = x14289 + z14290 = x14290 + z14291 = x14291 + z14292 = x14292 + z14293 = x14293 + z14294 = x14294 + z14295 = x14295 + z14296 = x14296 + z14297 = x14297 + z14298 = x14298 + z14299 = x14299 + z14300 = x14300 + z14301 = x14301 + z14302 = x14302 + z14303 = x14303 + z14304 = x14304 + z14305 = x14305 + z14306 = x14306 + z14307 = x14307 + z14308 = x14308 + z14309 = x14309 + z14310 = x14310 + z14311 = x14311 + z14312 = x14312 + z14313 = x14313 + z14314 = x14314 + z14315 = x14315 + z14316 = x14316 + z14317 = x14317 + z14318 = x14318 + z14319 = x14319 + z14320 = x14320 + z14321 = x14321 + z14322 = x14322 + z14323 = x14323 + z14324 = x14324 + z14325 = x14325 + z14326 = x14326 + z14327 = x14327 + z14328 = x14328 + z14329 = x14329 + z14330 = x14330 + z14331 = x14331 + z14332 = x14332 + z14333 = x14333 + z14334 = x14334 + z14335 = x14335 + z14336 = x14336 + z14337 = x14337 + z14338 = x14338 + z14339 = x14339 + z14340 = x14340 + z14341 = x14341 + z14342 = x14342 + z14343 = x14343 + z14344 = x14344 + z14345 = x14345 + z14346 = x14346 + z14347 = x14347 + z14348 = x14348 + z14349 = x14349 + z14350 = x14350 + z14351 = x14351 + z14352 = x14352 + z14353 = x14353 + z14354 = x14354 + z14355 = x14355 + z14356 = x14356 + z14357 = x14357 + z14358 = x14358 + z14359 = x14359 + z14360 = x14360 + z14361 = x14361 + z14362 = x14362 + z14363 = x14363 + z14364 = x14364 + z14365 = x14365 + z14366 = x14366 + z14367 = x14367 + z14368 = x14368 + z14369 = x14369 + z14370 = x14370 + z14371 = x14371 + z14372 = x14372 + z14373 = x14373 + z14374 = x14374 + z14375 = x14375 + z14376 = x14376 + z14377 = x14377 + z14378 = x14378 + z14379 = x14379 + z14380 = x14380 + z14381 = x14381 + z14382 = x14382 + z14383 = x14383 + z14384 = x14384 + z14385 = x14385 + z14386 = x14386 + z14387 = x14387 + z14388 = x14388 + z14389 = x14389 + z14390 = x14390 + z14391 = x14391 + z14392 = x14392 + z14393 = x14393 + z14394 = x14394 + z14395 = x14395 + z14396 = x14396 + z14397 = x14397 + z14398 = x14398 + z14399 = x14399 + z14400 = x14400 + z14401 = x14401 + z14402 = x14402 + z14403 = x14403 + z14404 = x14404 + z14405 = x14405 + z14406 = x14406 + z14407 = x14407 + z14408 = x14408 + z14409 = x14409 + z14410 = x14410 + z14411 = x14411 + z14412 = x14412 + z14413 = x14413 + z14414 = x14414 + z14415 = x14415 + z14416 = x14416 + z14417 = x14417 + z14418 = x14418 + z14419 = x14419 + z14420 = x14420 + z14421 = x14421 + z14422 = x14422 + z14423 = x14423 + z14424 = x14424 + z14425 = x14425 + z14426 = x14426 + z14427 = x14427 + z14428 = x14428 + z14429 = x14429 + z14430 = x14430 + z14431 = x14431 + z14432 = x14432 + z14433 = x14433 + z14434 = x14434 + z14435 = x14435 + z14436 = x14436 + z14437 = x14437 + z14438 = x14438 + z14439 = x14439 + z14440 = x14440 + z14441 = x14441 + z14442 = x14442 + z14443 = x14443 + z14444 = x14444 + z14445 = x14445 + z14446 = x14446 + z14447 = x14447 + z14448 = x14448 + z14449 = x14449 + z14450 = x14450 + z14451 = x14451 + z14452 = x14452 + z14453 = x14453 + z14454 = x14454 + z14455 = x14455 + z14456 = x14456 + z14457 = x14457 + z14458 = x14458 + z14459 = x14459 + z14460 = x14460 + z14461 = x14461 + z14462 = x14462 + z14463 = x14463 + z14464 = x14464 + z14465 = x14465 + z14466 = x14466 + z14467 = x14467 + z14468 = x14468 + z14469 = x14469 + z14470 = x14470 + z14471 = x14471 + z14472 = x14472 + z14473 = x14473 + z14474 = x14474 + z14475 = x14475 + z14476 = x14476 + z14477 = x14477 + z14478 = x14478 + z14479 = x14479 + z14480 = x14480 + z14481 = x14481 + z14482 = x14482 + z14483 = x14483 + z14484 = x14484 + z14485 = x14485 + z14486 = x14486 + z14487 = x14487 + z14488 = x14488 + z14489 = x14489 + z14490 = x14490 + z14491 = x14491 + z14492 = x14492 + z14493 = x14493 + z14494 = x14494 + z14495 = x14495 + z14496 = x14496 + z14497 = x14497 + z14498 = x14498 + z14499 = x14499 + z14500 = x14500 + z14501 = x14501 + z14502 = x14502 + z14503 = x14503 + z14504 = x14504 + z14505 = x14505 + z14506 = x14506 + z14507 = x14507 + z14508 = x14508 + z14509 = x14509 + z14510 = x14510 + z14511 = x14511 + z14512 = x14512 + z14513 = x14513 + z14514 = x14514 + z14515 = x14515 + z14516 = x14516 + z14517 = x14517 + z14518 = x14518 + z14519 = x14519 + z14520 = x14520 + z14521 = x14521 + z14522 = x14522 + z14523 = x14523 + z14524 = x14524 + z14525 = x14525 + z14526 = x14526 + z14527 = x14527 + z14528 = x14528 + z14529 = x14529 + z14530 = x14530 + z14531 = x14531 + z14532 = x14532 + z14533 = x14533 + z14534 = x14534 + z14535 = x14535 + z14536 = x14536 + z14537 = x14537 + z14538 = x14538 + z14539 = x14539 + z14540 = x14540 + z14541 = x14541 + z14542 = x14542 + z14543 = x14543 + z14544 = x14544 + z14545 = x14545 + z14546 = x14546 + z14547 = x14547 + z14548 = x14548 + z14549 = x14549 + z14550 = x14550 + z14551 = x14551 + z14552 = x14552 + z14553 = x14553 + z14554 = x14554 + z14555 = x14555 + z14556 = x14556 + z14557 = x14557 + z14558 = x14558 + z14559 = x14559 + z14560 = x14560 + z14561 = x14561 + z14562 = x14562 + z14563 = x14563 + z14564 = x14564 + z14565 = x14565 + z14566 = x14566 + z14567 = x14567 + z14568 = x14568 + z14569 = x14569 + z14570 = x14570 + z14571 = x14571 + z14572 = x14572 + z14573 = x14573 + z14574 = x14574 + z14575 = x14575 + z14576 = x14576 + z14577 = x14577 + z14578 = x14578 + z14579 = x14579 + z14580 = x14580 + z14581 = x14581 + z14582 = x14582 + z14583 = x14583 + z14584 = x14584 + z14585 = x14585 + z14586 = x14586 + z14587 = x14587 + z14588 = x14588 + z14589 = x14589 + z14590 = x14590 + z14591 = x14591 + z14592 = x14592 + z14593 = x14593 + z14594 = x14594 + z14595 = x14595 + z14596 = x14596 + z14597 = x14597 + z14598 = x14598 + z14599 = x14599 + z14600 = x14600 + z14601 = x14601 + z14602 = x14602 + z14603 = x14603 + z14604 = x14604 + z14605 = x14605 + z14606 = x14606 + z14607 = x14607 + z14608 = x14608 + z14609 = x14609 + z14610 = x14610 + z14611 = x14611 + z14612 = x14612 + z14613 = x14613 + z14614 = x14614 + z14615 = x14615 + z14616 = x14616 + z14617 = x14617 + z14618 = x14618 + z14619 = x14619 + z14620 = x14620 + z14621 = x14621 + z14622 = x14622 + z14623 = x14623 + z14624 = x14624 + z14625 = x14625 + z14626 = x14626 + z14627 = x14627 + z14628 = x14628 + z14629 = x14629 + z14630 = x14630 + z14631 = x14631 + z14632 = x14632 + z14633 = x14633 + z14634 = x14634 + z14635 = x14635 + z14636 = x14636 + z14637 = x14637 + z14638 = x14638 + z14639 = x14639 + z14640 = x14640 + z14641 = x14641 + z14642 = x14642 + z14643 = x14643 + z14644 = x14644 + z14645 = x14645 + z14646 = x14646 + z14647 = x14647 + z14648 = x14648 + z14649 = x14649 + z14650 = x14650 + z14651 = x14651 + z14652 = x14652 + z14653 = x14653 + z14654 = x14654 + z14655 = x14655 + z14656 = x14656 + z14657 = x14657 + z14658 = x14658 + z14659 = x14659 + z14660 = x14660 + z14661 = x14661 + z14662 = x14662 + z14663 = x14663 + z14664 = x14664 + z14665 = x14665 + z14666 = x14666 + z14667 = x14667 + z14668 = x14668 + z14669 = x14669 + z14670 = x14670 + z14671 = x14671 + z14672 = x14672 + z14673 = x14673 + z14674 = x14674 + z14675 = x14675 + z14676 = x14676 + z14677 = x14677 + z14678 = x14678 + z14679 = x14679 + z14680 = x14680 + z14681 = x14681 + z14682 = x14682 + z14683 = x14683 + z14684 = x14684 + z14685 = x14685 + z14686 = x14686 + z14687 = x14687 + z14688 = x14688 + z14689 = x14689 + z14690 = x14690 + z14691 = x14691 + z14692 = x14692 + z14693 = x14693 + z14694 = x14694 + z14695 = x14695 + z14696 = x14696 + z14697 = x14697 + z14698 = x14698 + z14699 = x14699 + z14700 = x14700 + z14701 = x14701 + z14702 = x14702 + z14703 = x14703 + z14704 = x14704 + z14705 = x14705 + z14706 = x14706 + z14707 = x14707 + z14708 = x14708 + z14709 = x14709 + z14710 = x14710 + z14711 = x14711 + z14712 = x14712 + z14713 = x14713 + z14714 = x14714 + z14715 = x14715 + z14716 = x14716 + z14717 = x14717 + z14718 = x14718 + z14719 = x14719 + z14720 = x14720 + z14721 = x14721 + z14722 = x14722 + z14723 = x14723 + z14724 = x14724 + z14725 = x14725 + z14726 = x14726 + z14727 = x14727 + z14728 = x14728 + z14729 = x14729 + z14730 = x14730 + z14731 = x14731 + z14732 = x14732 + z14733 = x14733 + z14734 = x14734 + z14735 = x14735 + z14736 = x14736 + z14737 = x14737 + z14738 = x14738 + z14739 = x14739 + z14740 = x14740 + z14741 = x14741 + z14742 = x14742 + z14743 = x14743 + z14744 = x14744 + z14745 = x14745 + z14746 = x14746 + z14747 = x14747 + z14748 = x14748 + z14749 = x14749 + z14750 = x14750 + z14751 = x14751 + z14752 = x14752 + z14753 = x14753 + z14754 = x14754 + z14755 = x14755 + z14756 = x14756 + z14757 = x14757 + z14758 = x14758 + z14759 = x14759 + z14760 = x14760 + z14761 = x14761 + z14762 = x14762 + z14763 = x14763 + z14764 = x14764 + z14765 = x14765 + z14766 = x14766 + z14767 = x14767 + z14768 = x14768 + z14769 = x14769 + z14770 = x14770 + z14771 = x14771 + z14772 = x14772 + z14773 = x14773 + z14774 = x14774 + z14775 = x14775 + z14776 = x14776 + z14777 = x14777 + z14778 = x14778 + z14779 = x14779 + z14780 = x14780 + z14781 = x14781 + z14782 = x14782 + z14783 = x14783 + z14784 = x14784 + z14785 = x14785 + z14786 = x14786 + z14787 = x14787 + z14788 = x14788 + z14789 = x14789 + z14790 = x14790 + z14791 = x14791 + z14792 = x14792 + z14793 = x14793 + z14794 = x14794 + z14795 = x14795 + z14796 = x14796 + z14797 = x14797 + z14798 = x14798 + z14799 = x14799 + z14800 = x14800 + z14801 = x14801 + z14802 = x14802 + z14803 = x14803 + z14804 = x14804 + z14805 = x14805 + z14806 = x14806 + z14807 = x14807 + z14808 = x14808 + z14809 = x14809 + z14810 = x14810 + z14811 = x14811 + z14812 = x14812 + z14813 = x14813 + z14814 = x14814 + z14815 = x14815 + z14816 = x14816 + z14817 = x14817 + z14818 = x14818 + z14819 = x14819 + z14820 = x14820 + z14821 = x14821 + z14822 = x14822 + z14823 = x14823 + z14824 = x14824 + z14825 = x14825 + z14826 = x14826 + z14827 = x14827 + z14828 = x14828 + z14829 = x14829 + z14830 = x14830 + z14831 = x14831 + z14832 = x14832 + z14833 = x14833 + z14834 = x14834 + z14835 = x14835 + z14836 = x14836 + z14837 = x14837 + z14838 = x14838 + z14839 = x14839 + z14840 = x14840 + z14841 = x14841 + z14842 = x14842 + z14843 = x14843 + z14844 = x14844 + z14845 = x14845 + z14846 = x14846 + z14847 = x14847 + z14848 = x14848 + z14849 = x14849 + z14850 = x14850 + z14851 = x14851 + z14852 = x14852 + z14853 = x14853 + z14854 = x14854 + z14855 = x14855 + z14856 = x14856 + z14857 = x14857 + z14858 = x14858 + z14859 = x14859 + z14860 = x14860 + z14861 = x14861 + z14862 = x14862 + z14863 = x14863 + z14864 = x14864 + z14865 = x14865 + z14866 = x14866 + z14867 = x14867 + z14868 = x14868 + z14869 = x14869 + z14870 = x14870 + z14871 = x14871 + z14872 = x14872 + z14873 = x14873 + z14874 = x14874 + z14875 = x14875 + z14876 = x14876 + z14877 = x14877 + z14878 = x14878 + z14879 = x14879 + z14880 = x14880 + z14881 = x14881 + z14882 = x14882 + z14883 = x14883 + z14884 = x14884 + z14885 = x14885 + z14886 = x14886 + z14887 = x14887 + z14888 = x14888 + z14889 = x14889 + z14890 = x14890 + z14891 = x14891 + z14892 = x14892 + z14893 = x14893 + z14894 = x14894 + z14895 = x14895 + z14896 = x14896 + z14897 = x14897 + z14898 = x14898 + z14899 = x14899 + z14900 = x14900 + z14901 = x14901 + z14902 = x14902 + z14903 = x14903 + z14904 = x14904 + z14905 = x14905 + z14906 = x14906 + z14907 = x14907 + z14908 = x14908 + z14909 = x14909 + z14910 = x14910 + z14911 = x14911 + z14912 = x14912 + z14913 = x14913 + z14914 = x14914 + z14915 = x14915 + z14916 = x14916 + z14917 = x14917 + z14918 = x14918 + z14919 = x14919 + z14920 = x14920 + z14921 = x14921 + z14922 = x14922 + z14923 = x14923 + z14924 = x14924 + z14925 = x14925 + z14926 = x14926 + z14927 = x14927 + z14928 = x14928 + z14929 = x14929 + z14930 = x14930 + z14931 = x14931 + z14932 = x14932 + z14933 = x14933 + z14934 = x14934 + z14935 = x14935 + z14936 = x14936 + z14937 = x14937 + z14938 = x14938 + z14939 = x14939 + z14940 = x14940 + z14941 = x14941 + z14942 = x14942 + z14943 = x14943 + z14944 = x14944 + z14945 = x14945 + z14946 = x14946 + z14947 = x14947 + z14948 = x14948 + z14949 = x14949 + z14950 = x14950 + z14951 = x14951 + z14952 = x14952 + z14953 = x14953 + z14954 = x14954 + z14955 = x14955 + z14956 = x14956 + z14957 = x14957 + z14958 = x14958 + z14959 = x14959 + z14960 = x14960 + z14961 = x14961 + z14962 = x14962 + z14963 = x14963 + z14964 = x14964 + z14965 = x14965 + z14966 = x14966 + z14967 = x14967 + z14968 = x14968 + z14969 = x14969 + z14970 = x14970 + z14971 = x14971 + z14972 = x14972 + z14973 = x14973 + z14974 = x14974 + z14975 = x14975 + z14976 = x14976 + z14977 = x14977 + z14978 = x14978 + z14979 = x14979 + z14980 = x14980 + z14981 = x14981 + z14982 = x14982 + z14983 = x14983 + z14984 = x14984 + z14985 = x14985 + z14986 = x14986 + z14987 = x14987 + z14988 = x14988 + z14989 = x14989 + z14990 = x14990 + z14991 = x14991 + z14992 = x14992 + z14993 = x14993 + z14994 = x14994 + z14995 = x14995 + z14996 = x14996 + z14997 = x14997 + z14998 = x14998 + z14999 = x14999 + z15000 = x15000 + z15001 = x15001 + z15002 = x15002 + z15003 = x15003 + z15004 = x15004 + z15005 = x15005 + z15006 = x15006 + z15007 = x15007 + z15008 = x15008 + z15009 = x15009 + z15010 = x15010 + z15011 = x15011 + z15012 = x15012 + z15013 = x15013 + z15014 = x15014 + z15015 = x15015 + z15016 = x15016 + z15017 = x15017 + z15018 = x15018 + z15019 = x15019 + z15020 = x15020 + z15021 = x15021 + z15022 = x15022 + z15023 = x15023 + z15024 = x15024 + z15025 = x15025 + z15026 = x15026 + z15027 = x15027 + z15028 = x15028 + z15029 = x15029 + z15030 = x15030 + z15031 = x15031 + z15032 = x15032 + z15033 = x15033 + z15034 = x15034 + z15035 = x15035 + z15036 = x15036 + z15037 = x15037 + z15038 = x15038 + z15039 = x15039 + z15040 = x15040 + z15041 = x15041 + z15042 = x15042 + z15043 = x15043 + z15044 = x15044 + z15045 = x15045 + z15046 = x15046 + z15047 = x15047 + z15048 = x15048 + z15049 = x15049 + z15050 = x15050 + z15051 = x15051 + z15052 = x15052 + z15053 = x15053 + z15054 = x15054 + z15055 = x15055 + z15056 = x15056 + z15057 = x15057 + z15058 = x15058 + z15059 = x15059 + z15060 = x15060 + z15061 = x15061 + z15062 = x15062 + z15063 = x15063 + z15064 = x15064 + z15065 = x15065 + z15066 = x15066 + z15067 = x15067 + z15068 = x15068 + z15069 = x15069 + z15070 = x15070 + z15071 = x15071 + z15072 = x15072 + z15073 = x15073 + z15074 = x15074 + z15075 = x15075 + z15076 = x15076 + z15077 = x15077 + z15078 = x15078 + z15079 = x15079 + z15080 = x15080 + z15081 = x15081 + z15082 = x15082 + z15083 = x15083 + z15084 = x15084 + z15085 = x15085 + z15086 = x15086 + z15087 = x15087 + z15088 = x15088 + z15089 = x15089 + z15090 = x15090 + z15091 = x15091 + z15092 = x15092 + z15093 = x15093 + z15094 = x15094 + z15095 = x15095 + z15096 = x15096 + z15097 = x15097 + z15098 = x15098 + z15099 = x15099 + z15100 = x15100 + z15101 = x15101 + z15102 = x15102 + z15103 = x15103 + z15104 = x15104 + z15105 = x15105 + z15106 = x15106 + z15107 = x15107 + z15108 = x15108 + z15109 = x15109 + z15110 = x15110 + z15111 = x15111 + z15112 = x15112 + z15113 = x15113 + z15114 = x15114 + z15115 = x15115 + z15116 = x15116 + z15117 = x15117 + z15118 = x15118 + z15119 = x15119 + z15120 = x15120 + z15121 = x15121 + z15122 = x15122 + z15123 = x15123 + z15124 = x15124 + z15125 = x15125 + z15126 = x15126 + z15127 = x15127 + z15128 = x15128 + z15129 = x15129 + z15130 = x15130 + z15131 = x15131 + z15132 = x15132 + z15133 = x15133 + z15134 = x15134 + z15135 = x15135 + z15136 = x15136 + z15137 = x15137 + z15138 = x15138 + z15139 = x15139 + z15140 = x15140 + z15141 = x15141 + z15142 = x15142 + z15143 = x15143 + z15144 = x15144 + z15145 = x15145 + z15146 = x15146 + z15147 = x15147 + z15148 = x15148 + z15149 = x15149 + z15150 = x15150 + z15151 = x15151 + z15152 = x15152 + z15153 = x15153 + z15154 = x15154 + z15155 = x15155 + z15156 = x15156 + z15157 = x15157 + z15158 = x15158 + z15159 = x15159 + z15160 = x15160 + z15161 = x15161 + z15162 = x15162 + z15163 = x15163 + z15164 = x15164 + z15165 = x15165 + z15166 = x15166 + z15167 = x15167 + z15168 = x15168 + z15169 = x15169 + z15170 = x15170 + z15171 = x15171 + z15172 = x15172 + z15173 = x15173 + z15174 = x15174 + z15175 = x15175 + z15176 = x15176 + z15177 = x15177 + z15178 = x15178 + z15179 = x15179 + z15180 = x15180 + z15181 = x15181 + z15182 = x15182 + z15183 = x15183 + z15184 = x15184 + z15185 = x15185 + z15186 = x15186 + z15187 = x15187 + z15188 = x15188 + z15189 = x15189 + z15190 = x15190 + z15191 = x15191 + z15192 = x15192 + z15193 = x15193 + z15194 = x15194 + z15195 = x15195 + z15196 = x15196 + z15197 = x15197 + z15198 = x15198 + z15199 = x15199 + z15200 = x15200 + z15201 = x15201 + z15202 = x15202 + z15203 = x15203 + z15204 = x15204 + z15205 = x15205 + z15206 = x15206 + z15207 = x15207 + z15208 = x15208 + z15209 = x15209 + z15210 = x15210 + z15211 = x15211 + z15212 = x15212 + z15213 = x15213 + z15214 = x15214 + z15215 = x15215 + z15216 = x15216 + z15217 = x15217 + z15218 = x15218 + z15219 = x15219 + z15220 = x15220 + z15221 = x15221 + z15222 = x15222 + z15223 = x15223 + z15224 = x15224 + z15225 = x15225 + z15226 = x15226 + z15227 = x15227 + z15228 = x15228 + z15229 = x15229 + z15230 = x15230 + z15231 = x15231 + z15232 = x15232 + z15233 = x15233 + z15234 = x15234 + z15235 = x15235 + z15236 = x15236 + z15237 = x15237 + z15238 = x15238 + z15239 = x15239 + z15240 = x15240 + z15241 = x15241 + z15242 = x15242 + z15243 = x15243 + z15244 = x15244 + z15245 = x15245 + z15246 = x15246 + z15247 = x15247 + z15248 = x15248 + z15249 = x15249 + z15250 = x15250 + z15251 = x15251 + z15252 = x15252 + z15253 = x15253 + z15254 = x15254 + z15255 = x15255 + z15256 = x15256 + z15257 = x15257 + z15258 = x15258 + z15259 = x15259 + z15260 = x15260 + z15261 = x15261 + z15262 = x15262 + z15263 = x15263 + z15264 = x15264 + z15265 = x15265 + z15266 = x15266 + z15267 = x15267 + z15268 = x15268 + z15269 = x15269 + z15270 = x15270 + z15271 = x15271 + z15272 = x15272 + z15273 = x15273 + z15274 = x15274 + z15275 = x15275 + z15276 = x15276 + z15277 = x15277 + z15278 = x15278 + z15279 = x15279 + z15280 = x15280 + z15281 = x15281 + z15282 = x15282 + z15283 = x15283 + z15284 = x15284 + z15285 = x15285 + z15286 = x15286 + z15287 = x15287 + z15288 = x15288 + z15289 = x15289 + z15290 = x15290 + z15291 = x15291 + z15292 = x15292 + z15293 = x15293 + z15294 = x15294 + z15295 = x15295 + z15296 = x15296 + z15297 = x15297 + z15298 = x15298 + z15299 = x15299 + z15300 = x15300 + z15301 = x15301 + z15302 = x15302 + z15303 = x15303 + z15304 = x15304 + z15305 = x15305 + z15306 = x15306 + z15307 = x15307 + z15308 = x15308 + z15309 = x15309 + z15310 = x15310 + z15311 = x15311 + z15312 = x15312 + z15313 = x15313 + z15314 = x15314 + z15315 = x15315 + z15316 = x15316 + z15317 = x15317 + z15318 = x15318 + z15319 = x15319 + z15320 = x15320 + z15321 = x15321 + z15322 = x15322 + z15323 = x15323 + z15324 = x15324 + z15325 = x15325 + z15326 = x15326 + z15327 = x15327 + z15328 = x15328 + z15329 = x15329 + z15330 = x15330 + z15331 = x15331 + z15332 = x15332 + z15333 = x15333 + z15334 = x15334 + z15335 = x15335 + z15336 = x15336 + z15337 = x15337 + z15338 = x15338 + z15339 = x15339 + z15340 = x15340 + z15341 = x15341 + z15342 = x15342 + z15343 = x15343 + z15344 = x15344 + z15345 = x15345 + z15346 = x15346 + z15347 = x15347 + z15348 = x15348 + z15349 = x15349 + z15350 = x15350 + z15351 = x15351 + z15352 = x15352 + z15353 = x15353 + z15354 = x15354 + z15355 = x15355 + z15356 = x15356 + z15357 = x15357 + z15358 = x15358 + z15359 = x15359 + z15360 = x15360 + z15361 = x15361 + z15362 = x15362 + z15363 = x15363 + z15364 = x15364 + z15365 = x15365 + z15366 = x15366 + z15367 = x15367 + z15368 = x15368 + z15369 = x15369 + z15370 = x15370 + z15371 = x15371 + z15372 = x15372 + z15373 = x15373 + z15374 = x15374 + z15375 = x15375 + z15376 = x15376 + z15377 = x15377 + z15378 = x15378 + z15379 = x15379 + z15380 = x15380 + z15381 = x15381 + z15382 = x15382 + z15383 = x15383 + z15384 = x15384 + z15385 = x15385 + z15386 = x15386 + z15387 = x15387 + z15388 = x15388 + z15389 = x15389 + z15390 = x15390 + z15391 = x15391 + z15392 = x15392 + z15393 = x15393 + z15394 = x15394 + z15395 = x15395 + z15396 = x15396 + z15397 = x15397 + z15398 = x15398 + z15399 = x15399 + z15400 = x15400 + z15401 = x15401 + z15402 = x15402 + z15403 = x15403 + z15404 = x15404 + z15405 = x15405 + z15406 = x15406 + z15407 = x15407 + z15408 = x15408 + z15409 = x15409 + z15410 = x15410 + z15411 = x15411 + z15412 = x15412 + z15413 = x15413 + z15414 = x15414 + z15415 = x15415 + z15416 = x15416 + z15417 = x15417 + z15418 = x15418 + z15419 = x15419 + z15420 = x15420 + z15421 = x15421 + z15422 = x15422 + z15423 = x15423 + z15424 = x15424 + z15425 = x15425 + z15426 = x15426 + z15427 = x15427 + z15428 = x15428 + z15429 = x15429 + z15430 = x15430 + z15431 = x15431 + z15432 = x15432 + z15433 = x15433 + z15434 = x15434 + z15435 = x15435 + z15436 = x15436 + z15437 = x15437 + z15438 = x15438 + z15439 = x15439 + z15440 = x15440 + z15441 = x15441 + z15442 = x15442 + z15443 = x15443 + z15444 = x15444 + z15445 = x15445 + z15446 = x15446 + z15447 = x15447 + z15448 = x15448 + z15449 = x15449 + z15450 = x15450 + z15451 = x15451 + z15452 = x15452 + z15453 = x15453 + z15454 = x15454 + z15455 = x15455 + z15456 = x15456 + z15457 = x15457 + z15458 = x15458 + z15459 = x15459 + z15460 = x15460 + z15461 = x15461 + z15462 = x15462 + z15463 = x15463 + z15464 = x15464 + z15465 = x15465 + z15466 = x15466 + z15467 = x15467 + z15468 = x15468 + z15469 = x15469 + z15470 = x15470 + z15471 = x15471 + z15472 = x15472 + z15473 = x15473 + z15474 = x15474 + z15475 = x15475 + z15476 = x15476 + z15477 = x15477 + z15478 = x15478 + z15479 = x15479 + z15480 = x15480 + z15481 = x15481 + z15482 = x15482 + z15483 = x15483 + z15484 = x15484 + z15485 = x15485 + z15486 = x15486 + z15487 = x15487 + z15488 = x15488 + z15489 = x15489 + z15490 = x15490 + z15491 = x15491 + z15492 = x15492 + z15493 = x15493 + z15494 = x15494 + z15495 = x15495 + z15496 = x15496 + z15497 = x15497 + z15498 = x15498 + z15499 = x15499 + z15500 = x15500 + z15501 = x15501 + z15502 = x15502 + z15503 = x15503 + z15504 = x15504 + z15505 = x15505 + z15506 = x15506 + z15507 = x15507 + z15508 = x15508 + z15509 = x15509 + z15510 = x15510 + z15511 = x15511 + z15512 = x15512 + z15513 = x15513 + z15514 = x15514 + z15515 = x15515 + z15516 = x15516 + z15517 = x15517 + z15518 = x15518 + z15519 = x15519 + z15520 = x15520 + z15521 = x15521 + z15522 = x15522 + z15523 = x15523 + z15524 = x15524 + z15525 = x15525 + z15526 = x15526 + z15527 = x15527 + z15528 = x15528 + z15529 = x15529 + z15530 = x15530 + z15531 = x15531 + z15532 = x15532 + z15533 = x15533 + z15534 = x15534 + z15535 = x15535 + z15536 = x15536 + z15537 = x15537 + z15538 = x15538 + z15539 = x15539 + z15540 = x15540 + z15541 = x15541 + z15542 = x15542 + z15543 = x15543 + z15544 = x15544 + z15545 = x15545 + z15546 = x15546 + z15547 = x15547 + z15548 = x15548 + z15549 = x15549 + z15550 = x15550 + z15551 = x15551 + z15552 = x15552 + z15553 = x15553 + z15554 = x15554 + z15555 = x15555 + z15556 = x15556 + z15557 = x15557 + z15558 = x15558 + z15559 = x15559 + z15560 = x15560 + z15561 = x15561 + z15562 = x15562 + z15563 = x15563 + z15564 = x15564 + z15565 = x15565 + z15566 = x15566 + z15567 = x15567 + z15568 = x15568 + z15569 = x15569 + z15570 = x15570 + z15571 = x15571 + z15572 = x15572 + z15573 = x15573 + z15574 = x15574 + z15575 = x15575 + z15576 = x15576 + z15577 = x15577 + z15578 = x15578 + z15579 = x15579 + z15580 = x15580 + z15581 = x15581 + z15582 = x15582 + z15583 = x15583 + z15584 = x15584 + z15585 = x15585 + z15586 = x15586 + z15587 = x15587 + z15588 = x15588 + z15589 = x15589 + z15590 = x15590 + z15591 = x15591 + z15592 = x15592 + z15593 = x15593 + z15594 = x15594 + z15595 = x15595 + z15596 = x15596 + z15597 = x15597 + z15598 = x15598 + z15599 = x15599 + z15600 = x15600 + z15601 = x15601 + z15602 = x15602 + z15603 = x15603 + z15604 = x15604 + z15605 = x15605 + z15606 = x15606 + z15607 = x15607 + z15608 = x15608 + z15609 = x15609 + z15610 = x15610 + z15611 = x15611 + z15612 = x15612 + z15613 = x15613 + z15614 = x15614 + z15615 = x15615 + z15616 = x15616 + z15617 = x15617 + z15618 = x15618 + z15619 = x15619 + z15620 = x15620 + z15621 = x15621 + z15622 = x15622 + z15623 = x15623 + z15624 = x15624 + z15625 = x15625 + z15626 = x15626 + z15627 = x15627 + z15628 = x15628 + z15629 = x15629 + z15630 = x15630 + z15631 = x15631 + z15632 = x15632 + z15633 = x15633 + z15634 = x15634 + z15635 = x15635 + z15636 = x15636 + z15637 = x15637 + z15638 = x15638 + z15639 = x15639 + z15640 = x15640 + z15641 = x15641 + z15642 = x15642 + z15643 = x15643 + z15644 = x15644 + z15645 = x15645 + z15646 = x15646 + z15647 = x15647 + z15648 = x15648 + z15649 = x15649 + z15650 = x15650 + z15651 = x15651 + z15652 = x15652 + z15653 = x15653 + z15654 = x15654 + z15655 = x15655 + z15656 = x15656 + z15657 = x15657 + z15658 = x15658 + z15659 = x15659 + z15660 = x15660 + z15661 = x15661 + z15662 = x15662 + z15663 = x15663 + z15664 = x15664 + z15665 = x15665 + z15666 = x15666 + z15667 = x15667 + z15668 = x15668 + z15669 = x15669 + z15670 = x15670 + z15671 = x15671 + z15672 = x15672 + z15673 = x15673 + z15674 = x15674 + z15675 = x15675 + z15676 = x15676 + z15677 = x15677 + z15678 = x15678 + z15679 = x15679 + z15680 = x15680 + z15681 = x15681 + z15682 = x15682 + z15683 = x15683 + z15684 = x15684 + z15685 = x15685 + z15686 = x15686 + z15687 = x15687 + z15688 = x15688 + z15689 = x15689 + z15690 = x15690 + z15691 = x15691 + z15692 = x15692 + z15693 = x15693 + z15694 = x15694 + z15695 = x15695 + z15696 = x15696 + z15697 = x15697 + z15698 = x15698 + z15699 = x15699 + z15700 = x15700 + z15701 = x15701 + z15702 = x15702 + z15703 = x15703 + z15704 = x15704 + z15705 = x15705 + z15706 = x15706 + z15707 = x15707 + z15708 = x15708 + z15709 = x15709 + z15710 = x15710 + z15711 = x15711 + z15712 = x15712 + z15713 = x15713 + z15714 = x15714 + z15715 = x15715 + z15716 = x15716 + z15717 = x15717 + z15718 = x15718 + z15719 = x15719 + z15720 = x15720 + z15721 = x15721 + z15722 = x15722 + z15723 = x15723 + z15724 = x15724 + z15725 = x15725 + z15726 = x15726 + z15727 = x15727 + z15728 = x15728 + z15729 = x15729 + z15730 = x15730 + z15731 = x15731 + z15732 = x15732 + z15733 = x15733 + z15734 = x15734 + z15735 = x15735 + z15736 = x15736 + z15737 = x15737 + z15738 = x15738 + z15739 = x15739 + z15740 = x15740 + z15741 = x15741 + z15742 = x15742 + z15743 = x15743 + z15744 = x15744 + z15745 = x15745 + z15746 = x15746 + z15747 = x15747 + z15748 = x15748 + z15749 = x15749 + z15750 = x15750 + z15751 = x15751 + z15752 = x15752 + z15753 = x15753 + z15754 = x15754 + z15755 = x15755 + z15756 = x15756 + z15757 = x15757 + z15758 = x15758 + z15759 = x15759 + z15760 = x15760 + z15761 = x15761 + z15762 = x15762 + z15763 = x15763 + z15764 = x15764 + z15765 = x15765 + z15766 = x15766 + z15767 = x15767 + z15768 = x15768 + z15769 = x15769 + z15770 = x15770 + z15771 = x15771 + z15772 = x15772 + z15773 = x15773 + z15774 = x15774 + z15775 = x15775 + z15776 = x15776 + z15777 = x15777 + z15778 = x15778 + z15779 = x15779 + z15780 = x15780 + z15781 = x15781 + z15782 = x15782 + z15783 = x15783 + z15784 = x15784 + z15785 = x15785 + z15786 = x15786 + z15787 = x15787 + z15788 = x15788 + z15789 = x15789 + z15790 = x15790 + z15791 = x15791 + z15792 = x15792 + z15793 = x15793 + z15794 = x15794 + z15795 = x15795 + z15796 = x15796 + z15797 = x15797 + z15798 = x15798 + z15799 = x15799 + z15800 = x15800 + z15801 = x15801 + z15802 = x15802 + z15803 = x15803 + z15804 = x15804 + z15805 = x15805 + z15806 = x15806 + z15807 = x15807 + z15808 = x15808 + z15809 = x15809 + z15810 = x15810 + z15811 = x15811 + z15812 = x15812 + z15813 = x15813 + z15814 = x15814 + z15815 = x15815 + z15816 = x15816 + z15817 = x15817 + z15818 = x15818 + z15819 = x15819 + z15820 = x15820 + z15821 = x15821 + z15822 = x15822 + z15823 = x15823 + z15824 = x15824 + z15825 = x15825 + z15826 = x15826 + z15827 = x15827 + z15828 = x15828 + z15829 = x15829 + z15830 = x15830 + z15831 = x15831 + z15832 = x15832 + z15833 = x15833 + z15834 = x15834 + z15835 = x15835 + z15836 = x15836 + z15837 = x15837 + z15838 = x15838 + z15839 = x15839 + z15840 = x15840 + z15841 = x15841 + z15842 = x15842 + z15843 = x15843 + z15844 = x15844 + z15845 = x15845 + z15846 = x15846 + z15847 = x15847 + z15848 = x15848 + z15849 = x15849 + z15850 = x15850 + z15851 = x15851 + z15852 = x15852 + z15853 = x15853 + z15854 = x15854 + z15855 = x15855 + z15856 = x15856 + z15857 = x15857 + z15858 = x15858 + z15859 = x15859 + z15860 = x15860 + z15861 = x15861 + z15862 = x15862 + z15863 = x15863 + z15864 = x15864 + z15865 = x15865 + z15866 = x15866 + z15867 = x15867 + z15868 = x15868 + z15869 = x15869 + z15870 = x15870 + z15871 = x15871 + z15872 = x15872 + z15873 = x15873 + z15874 = x15874 + z15875 = x15875 + z15876 = x15876 + z15877 = x15877 + z15878 = x15878 + z15879 = x15879 + z15880 = x15880 + z15881 = x15881 + z15882 = x15882 + z15883 = x15883 + z15884 = x15884 + z15885 = x15885 + z15886 = x15886 + z15887 = x15887 + z15888 = x15888 + z15889 = x15889 + z15890 = x15890 + z15891 = x15891 + z15892 = x15892 + z15893 = x15893 + z15894 = x15894 + z15895 = x15895 + z15896 = x15896 + z15897 = x15897 + z15898 = x15898 + z15899 = x15899 + z15900 = x15900 + z15901 = x15901 + z15902 = x15902 + z15903 = x15903 + z15904 = x15904 + z15905 = x15905 + z15906 = x15906 + z15907 = x15907 + z15908 = x15908 + z15909 = x15909 + z15910 = x15910 + z15911 = x15911 + z15912 = x15912 + z15913 = x15913 + z15914 = x15914 + z15915 = x15915 + z15916 = x15916 + z15917 = x15917 + z15918 = x15918 + z15919 = x15919 + z15920 = x15920 + z15921 = x15921 + z15922 = x15922 + z15923 = x15923 + z15924 = x15924 + z15925 = x15925 + z15926 = x15926 + z15927 = x15927 + z15928 = x15928 + z15929 = x15929 + z15930 = x15930 + z15931 = x15931 + z15932 = x15932 + z15933 = x15933 + z15934 = x15934 + z15935 = x15935 + z15936 = x15936 + z15937 = x15937 + z15938 = x15938 + z15939 = x15939 + z15940 = x15940 + z15941 = x15941 + z15942 = x15942 + z15943 = x15943 + z15944 = x15944 + z15945 = x15945 + z15946 = x15946 + z15947 = x15947 + z15948 = x15948 + z15949 = x15949 + z15950 = x15950 + z15951 = x15951 + z15952 = x15952 + z15953 = x15953 + z15954 = x15954 + z15955 = x15955 + z15956 = x15956 + z15957 = x15957 + z15958 = x15958 + z15959 = x15959 + z15960 = x15960 + z15961 = x15961 + z15962 = x15962 + z15963 = x15963 + z15964 = x15964 + z15965 = x15965 + z15966 = x15966 + z15967 = x15967 + z15968 = x15968 + z15969 = x15969 + z15970 = x15970 + z15971 = x15971 + z15972 = x15972 + z15973 = x15973 + z15974 = x15974 + z15975 = x15975 + z15976 = x15976 + z15977 = x15977 + z15978 = x15978 + z15979 = x15979 + z15980 = x15980 + z15981 = x15981 + z15982 = x15982 + z15983 = x15983 + z15984 = x15984 + z15985 = x15985 + z15986 = x15986 + z15987 = x15987 + z15988 = x15988 + z15989 = x15989 + z15990 = x15990 + z15991 = x15991 + z15992 = x15992 + z15993 = x15993 + z15994 = x15994 + z15995 = x15995 + z15996 = x15996 + z15997 = x15997 + z15998 = x15998 + z15999 = x15999 + z16000 = x16000 + z16001 = x16001 + z16002 = x16002 + z16003 = x16003 + z16004 = x16004 + z16005 = x16005 + z16006 = x16006 + z16007 = x16007 + z16008 = x16008 + z16009 = x16009 + z16010 = x16010 + z16011 = x16011 + z16012 = x16012 + z16013 = x16013 + z16014 = x16014 + z16015 = x16015 + z16016 = x16016 + z16017 = x16017 + z16018 = x16018 + z16019 = x16019 + z16020 = x16020 + z16021 = x16021 + z16022 = x16022 + z16023 = x16023 + z16024 = x16024 + z16025 = x16025 + z16026 = x16026 + z16027 = x16027 + z16028 = x16028 + z16029 = x16029 + z16030 = x16030 + z16031 = x16031 + z16032 = x16032 + z16033 = x16033 + z16034 = x16034 + z16035 = x16035 + z16036 = x16036 + z16037 = x16037 + z16038 = x16038 + z16039 = x16039 + z16040 = x16040 + z16041 = x16041 + z16042 = x16042 + z16043 = x16043 + z16044 = x16044 + z16045 = x16045 + z16046 = x16046 + z16047 = x16047 + z16048 = x16048 + z16049 = x16049 + z16050 = x16050 + z16051 = x16051 + z16052 = x16052 + z16053 = x16053 + z16054 = x16054 + z16055 = x16055 + z16056 = x16056 + z16057 = x16057 + z16058 = x16058 + z16059 = x16059 + z16060 = x16060 + z16061 = x16061 + z16062 = x16062 + z16063 = x16063 + z16064 = x16064 + z16065 = x16065 + z16066 = x16066 + z16067 = x16067 + z16068 = x16068 + z16069 = x16069 + z16070 = x16070 + z16071 = x16071 + z16072 = x16072 + z16073 = x16073 + z16074 = x16074 + z16075 = x16075 + z16076 = x16076 + z16077 = x16077 + z16078 = x16078 + z16079 = x16079 + z16080 = x16080 + z16081 = x16081 + z16082 = x16082 + z16083 = x16083 + z16084 = x16084 + z16085 = x16085 + z16086 = x16086 + z16087 = x16087 + z16088 = x16088 + z16089 = x16089 + z16090 = x16090 + z16091 = x16091 + z16092 = x16092 + z16093 = x16093 + z16094 = x16094 + z16095 = x16095 + z16096 = x16096 + z16097 = x16097 + z16098 = x16098 + z16099 = x16099 + z16100 = x16100 + z16101 = x16101 + z16102 = x16102 + z16103 = x16103 + z16104 = x16104 + z16105 = x16105 + z16106 = x16106 + z16107 = x16107 + z16108 = x16108 + z16109 = x16109 + z16110 = x16110 + z16111 = x16111 + z16112 = x16112 + z16113 = x16113 + z16114 = x16114 + z16115 = x16115 + z16116 = x16116 + z16117 = x16117 + z16118 = x16118 + z16119 = x16119 + z16120 = x16120 + z16121 = x16121 + z16122 = x16122 + z16123 = x16123 + z16124 = x16124 + z16125 = x16125 + z16126 = x16126 + z16127 = x16127 + z16128 = x16128 + z16129 = x16129 + z16130 = x16130 + z16131 = x16131 + z16132 = x16132 + z16133 = x16133 + z16134 = x16134 + z16135 = x16135 + z16136 = x16136 + z16137 = x16137 + z16138 = x16138 + z16139 = x16139 + z16140 = x16140 + z16141 = x16141 + z16142 = x16142 + z16143 = x16143 + z16144 = x16144 + z16145 = x16145 + z16146 = x16146 + z16147 = x16147 + z16148 = x16148 + z16149 = x16149 + z16150 = x16150 + z16151 = x16151 + z16152 = x16152 + z16153 = x16153 + z16154 = x16154 + z16155 = x16155 + z16156 = x16156 + z16157 = x16157 + z16158 = x16158 + z16159 = x16159 + z16160 = x16160 + z16161 = x16161 + z16162 = x16162 + z16163 = x16163 + z16164 = x16164 + z16165 = x16165 + z16166 = x16166 + z16167 = x16167 + z16168 = x16168 + z16169 = x16169 + z16170 = x16170 + z16171 = x16171 + z16172 = x16172 + z16173 = x16173 + z16174 = x16174 + z16175 = x16175 + z16176 = x16176 + z16177 = x16177 + z16178 = x16178 + z16179 = x16179 + z16180 = x16180 + z16181 = x16181 + z16182 = x16182 + z16183 = x16183 + z16184 = x16184 + z16185 = x16185 + z16186 = x16186 + z16187 = x16187 + z16188 = x16188 + z16189 = x16189 + z16190 = x16190 + z16191 = x16191 + z16192 = x16192 + z16193 = x16193 + z16194 = x16194 + z16195 = x16195 + z16196 = x16196 + z16197 = x16197 + z16198 = x16198 + z16199 = x16199 + z16200 = x16200 + z16201 = x16201 + z16202 = x16202 + z16203 = x16203 + z16204 = x16204 + z16205 = x16205 + z16206 = x16206 + z16207 = x16207 + z16208 = x16208 + z16209 = x16209 + z16210 = x16210 + z16211 = x16211 + z16212 = x16212 + z16213 = x16213 + z16214 = x16214 + z16215 = x16215 + z16216 = x16216 + z16217 = x16217 + z16218 = x16218 + z16219 = x16219 + z16220 = x16220 + z16221 = x16221 + z16222 = x16222 + z16223 = x16223 + z16224 = x16224 + z16225 = x16225 + z16226 = x16226 + z16227 = x16227 + z16228 = x16228 + z16229 = x16229 + z16230 = x16230 + z16231 = x16231 + z16232 = x16232 + z16233 = x16233 + z16234 = x16234 + z16235 = x16235 + z16236 = x16236 + z16237 = x16237 + z16238 = x16238 + z16239 = x16239 + z16240 = x16240 + z16241 = x16241 + z16242 = x16242 + z16243 = x16243 + z16244 = x16244 + z16245 = x16245 + z16246 = x16246 + z16247 = x16247 + z16248 = x16248 + z16249 = x16249 + z16250 = x16250 + z16251 = x16251 + z16252 = x16252 + z16253 = x16253 + z16254 = x16254 + z16255 = x16255 + z16256 = x16256 + z16257 = x16257 + z16258 = x16258 + z16259 = x16259 + z16260 = x16260 + z16261 = x16261 + z16262 = x16262 + z16263 = x16263 + z16264 = x16264 + z16265 = x16265 + z16266 = x16266 + z16267 = x16267 + z16268 = x16268 + z16269 = x16269 + z16270 = x16270 + z16271 = x16271 + z16272 = x16272 + z16273 = x16273 + z16274 = x16274 + z16275 = x16275 + z16276 = x16276 + z16277 = x16277 + z16278 = x16278 + z16279 = x16279 + z16280 = x16280 + z16281 = x16281 + z16282 = x16282 + z16283 = x16283 + z16284 = x16284 + z16285 = x16285 + z16286 = x16286 + z16287 = x16287 + z16288 = x16288 + z16289 = x16289 + z16290 = x16290 + z16291 = x16291 + z16292 = x16292 + z16293 = x16293 + z16294 = x16294 + z16295 = x16295 + z16296 = x16296 + z16297 = x16297 + z16298 = x16298 + z16299 = x16299 + z16300 = x16300 + z16301 = x16301 + z16302 = x16302 + z16303 = x16303 + z16304 = x16304 + z16305 = x16305 + z16306 = x16306 + z16307 = x16307 + z16308 = x16308 + z16309 = x16309 + z16310 = x16310 + z16311 = x16311 + z16312 = x16312 + z16313 = x16313 + z16314 = x16314 + z16315 = x16315 + z16316 = x16316 + z16317 = x16317 + z16318 = x16318 + z16319 = x16319 + z16320 = x16320 + z16321 = x16321 + z16322 = x16322 + z16323 = x16323 + z16324 = x16324 + z16325 = x16325 + z16326 = x16326 + z16327 = x16327 + z16328 = x16328 + z16329 = x16329 + z16330 = x16330 + z16331 = x16331 + z16332 = x16332 + z16333 = x16333 + z16334 = x16334 + z16335 = x16335 + z16336 = x16336 + z16337 = x16337 + z16338 = x16338 + z16339 = x16339 + z16340 = x16340 + z16341 = x16341 + z16342 = x16342 + z16343 = x16343 + z16344 = x16344 + z16345 = x16345 + z16346 = x16346 + z16347 = x16347 + z16348 = x16348 + z16349 = x16349 + z16350 = x16350 + z16351 = x16351 + z16352 = x16352 + z16353 = x16353 + z16354 = x16354 + z16355 = x16355 + z16356 = x16356 + z16357 = x16357 + z16358 = x16358 + z16359 = x16359 + z16360 = x16360 + z16361 = x16361 + z16362 = x16362 + z16363 = x16363 + z16364 = x16364 + z16365 = x16365 + z16366 = x16366 + z16367 = x16367 + z16368 = x16368 + z16369 = x16369 + z16370 = x16370 + z16371 = x16371 + z16372 = x16372 + z16373 = x16373 + z16374 = x16374 + z16375 = x16375 + z16376 = x16376 + z16377 = x16377 + z16378 = x16378 + z16379 = x16379 + z16380 = x16380 + z16381 = x16381 + z16382 = x16382 + z16383 = x16383 + z16384 = x16384 + z16385 = x16385 + z16386 = x16386 + z16387 = x16387 + z16388 = x16388 + z16389 = x16389 + z16390 = x16390 + z16391 = x16391 + z16392 = x16392 + z16393 = x16393 + z16394 = x16394 + z16395 = x16395 + z16396 = x16396 + z16397 = x16397 + z16398 = x16398 + z16399 = x16399 + z16400 = x16400 + z16401 = x16401 + z16402 = x16402 + z16403 = x16403 + z16404 = x16404 + z16405 = x16405 + z16406 = x16406 + z16407 = x16407 + z16408 = x16408 + z16409 = x16409 + z16410 = x16410 + z16411 = x16411 + z16412 = x16412 + z16413 = x16413 + z16414 = x16414 + z16415 = x16415 + z16416 = x16416 + z16417 = x16417 + z16418 = x16418 + z16419 = x16419 + z16420 = x16420 + z16421 = x16421 + z16422 = x16422 + z16423 = x16423 + z16424 = x16424 + z16425 = x16425 + z16426 = x16426 + z16427 = x16427 + z16428 = x16428 + z16429 = x16429 + z16430 = x16430 + z16431 = x16431 + z16432 = x16432 + z16433 = x16433 + z16434 = x16434 + z16435 = x16435 + z16436 = x16436 + z16437 = x16437 + z16438 = x16438 + z16439 = x16439 + z16440 = x16440 + z16441 = x16441 + z16442 = x16442 + z16443 = x16443 + z16444 = x16444 + z16445 = x16445 + z16446 = x16446 + z16447 = x16447 + z16448 = x16448 + z16449 = x16449 + z16450 = x16450 + z16451 = x16451 + z16452 = x16452 + z16453 = x16453 + z16454 = x16454 + z16455 = x16455 + z16456 = x16456 + z16457 = x16457 + z16458 = x16458 + z16459 = x16459 + z16460 = x16460 + z16461 = x16461 + z16462 = x16462 + z16463 = x16463 + z16464 = x16464 + z16465 = x16465 + z16466 = x16466 + z16467 = x16467 + z16468 = x16468 + z16469 = x16469 + z16470 = x16470 + z16471 = x16471 + z16472 = x16472 + z16473 = x16473 + z16474 = x16474 + z16475 = x16475 + z16476 = x16476 + z16477 = x16477 + z16478 = x16478 + z16479 = x16479 + z16480 = x16480 } diff --git a/test/fixedbugs/bug388.go b/test/fixedbugs/bug388.go index a060c9fd..0524534b 100644 --- a/test/fixedbugs/bug388.go +++ b/test/fixedbugs/bug388.go @@ -9,7 +9,7 @@ package main import "runtime" -func foo(runtime.UintType, i int) { // ERROR "cannot declare name runtime.UintType|mixed named and unnamed|undefined identifier" +func foo(runtime.UintType, i int) { // ERROR "cannot declare name runtime.UintType|missing parameter name|undefined identifier" println(i, runtime.UintType) // GCCGO_ERROR "undefined identifier" } diff --git a/test/fixedbugs/issue11610.go b/test/fixedbugs/issue11610.go index 8d68c98f..ec4d8e84 100644 --- a/test/fixedbugs/issue11610.go +++ b/test/fixedbugs/issue11610.go @@ -10,7 +10,7 @@ package a var? // ERROR "invalid character U\+003F '\?'|invalid character 0x3f in input file" -var x int // ERROR "unexpected var|expected identifier|expected type" +var x int // ERROR "unexpected keyword var|expected identifier|expected type" func main() { } diff --git a/test/fixedbugs/issue16241.go b/test/fixedbugs/issue16241.go new file mode 100644 index 00000000..a88dc9a4 --- /dev/null +++ b/test/fixedbugs/issue16241.go @@ -0,0 +1,79 @@ +// errorcheck -0 -m -l + +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo + +import "sync/atomic" + +func AddInt32(x *int32) { // ERROR "x does not escape$" + atomic.AddInt32(x, 42) +} +func AddUint32(x *uint32) { // ERROR "x does not escape$" + atomic.AddUint32(x, 42) +} +func AddUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.AddUintptr(x, 42) +} + +func AndInt32(x *int32) { // ERROR "x does not escape$" + atomic.AndInt32(x, 42) +} +func AndUint32(x *uint32) { // ERROR "x does not escape$" + atomic.AndUint32(x, 42) +} +func AndUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.AndUintptr(x, 42) +} + +func CompareAndSwapInt32(x *int32) { // ERROR "x does not escape$" + atomic.CompareAndSwapInt32(x, 42, 42) +} +func CompareAndSwapUint32(x *uint32) { // ERROR "x does not escape$" + atomic.CompareAndSwapUint32(x, 42, 42) +} +func CompareAndSwapUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.CompareAndSwapUintptr(x, 42, 42) +} + +func LoadInt32(x *int32) { // ERROR "x does not escape$" + atomic.LoadInt32(x) +} +func LoadUint32(x *uint32) { // ERROR "x does not escape$" + atomic.LoadUint32(x) +} +func LoadUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.LoadUintptr(x) +} + +func OrInt32(x *int32) { // ERROR "x does not escape$" + atomic.OrInt32(x, 42) +} +func OrUint32(x *uint32) { // ERROR "x does not escape$" + atomic.OrUint32(x, 42) +} +func OrUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.OrUintptr(x, 42) +} + +func StoreInt32(x *int32) { // ERROR "x does not escape$" + atomic.StoreInt32(x, 42) +} +func StoreUint32(x *uint32) { // ERROR "x does not escape$" + atomic.StoreUint32(x, 42) +} +func StoreUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.StoreUintptr(x, 42) +} + +func SwapInt32(x *int32) { // ERROR "x does not escape$" + atomic.SwapInt32(x, 42) +} +func SwapUint32(x *uint32) { // ERROR "x does not escape$" + atomic.SwapUint32(x, 42) +} +func SwapUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.SwapUintptr(x, 42) +} diff --git a/test/fixedbugs/issue16241_64.go b/test/fixedbugs/issue16241_64.go new file mode 100644 index 00000000..468444bd --- /dev/null +++ b/test/fixedbugs/issue16241_64.go @@ -0,0 +1,60 @@ +//go:build !(386 || arm || mips || mipsle) + +// errorcheck -0 -m -l + +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo + +import "sync/atomic" + +func AddInt64(x *int64) { // ERROR "x does not escape$" + atomic.AddInt64(x, 42) +} +func AddUint64(x *uint64) { // ERROR "x does not escape$" + atomic.AddUint64(x, 42) +} + +func AndInt64(x *int64) { // ERROR "x does not escape$" + atomic.AndInt64(x, 42) +} +func AndUint64(x *uint64) { // ERROR "x does not escape$" + atomic.AndUint64(x, 42) +} + +func CompareAndSwapInt64(x *int64) { // ERROR "x does not escape$" + atomic.CompareAndSwapInt64(x, 42, 42) +} +func CompareAndSwapUint64(x *uint64) { // ERROR "x does not escape$" + atomic.CompareAndSwapUint64(x, 42, 42) +} + +func LoadInt64(x *int64) { // ERROR "x does not escape$" + atomic.LoadInt64(x) +} +func LoadUint64(x *uint64) { // ERROR "x does not escape$" + atomic.LoadUint64(x) +} + +func OrInt64(x *int64) { // ERROR "x does not escape$" + atomic.OrInt64(x, 42) +} +func OrUint64(x *uint64) { // ERROR "x does not escape$" + atomic.OrUint64(x, 42) +} + +func StoreInt64(x *int64) { // ERROR "x does not escape$" + atomic.StoreInt64(x, 42) +} +func StoreUint64(x *uint64) { // ERROR "x does not escape$" + atomic.StoreUint64(x, 42) +} + +func SwapInt64(x *int64) { // ERROR "x does not escape$" + atomic.SwapInt64(x, 42) +} +func SwapUint64(x *uint64) { // ERROR "x does not escape$" + atomic.SwapUint64(x, 42) +} diff --git a/test/fixedbugs/issue20027.go b/test/fixedbugs/issue20027.go new file mode 100644 index 00000000..e93b359a --- /dev/null +++ b/test/fixedbugs/issue20027.go @@ -0,0 +1,13 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +var _ chan [0x2FFFF]byte // ERROR "channel element type too large" +var _ = make(chan [0x2FFFF]byte) // ERROR "channel element type too large" + +var c1 chan [0x2FFFF]byte // ERROR "channel element type too large" +var c2 = make(chan [0x2FFFF]byte) // ERROR "channel element type too large" diff --git a/test/fixedbugs/issue24755.go b/test/fixedbugs/issue24755.go new file mode 100644 index 00000000..705b15fb --- /dev/null +++ b/test/fixedbugs/issue24755.go @@ -0,0 +1,19 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type I interface { + F() +} + +type T struct { +} + +const _ = I((*T)(nil)) // ERROR "is not constant" + +func (*T) F() { +} diff --git a/test/fixedbugs/issue39292.go b/test/fixedbugs/issue39292.go index 7dac2e5f..9a4840d0 100644 --- a/test/fixedbugs/issue39292.go +++ b/test/fixedbugs/issue39292.go @@ -27,3 +27,12 @@ func z() { z := t{&i}.f // ERROR "t{...}.f escapes to heap" z() } + +// Should match cmd/compile/internal/ir/cfg.go:MaxStackVarSize. +const maxStack = 128 * 1024 + +func w(i int) byte { + var x [maxStack]byte + var y [maxStack + 1]byte // ERROR "moved to heap: y" + return x[i] + y[i] +} diff --git a/test/fixedbugs/issue41575.go b/test/fixedbugs/issue41575.go index 45687303..2eed37a9 100644 --- a/test/fixedbugs/issue41575.go +++ b/test/fixedbugs/issue41575.go @@ -6,7 +6,7 @@ package p -type T1 struct { // ERROR "invalid recursive type T1\n\tLINE: T1 refers to\n\tLINE+4: T2 refers to\n\tLINE: T1$|invalid recursive type" +type T1 struct { // ERROR "invalid recursive type T1\n.*T1 refers to T2\n.*T2 refers to T1|invalid recursive type" f2 T2 } @@ -15,21 +15,21 @@ type T2 struct { // GCCGO_ERROR "invalid recursive type" } type a b // GCCGO_ERROR "invalid recursive type" -type b c // ERROR "invalid recursive type b\n\tLINE: b refers to\n\tLINE+1: c refers to\n\tLINE: b$|invalid recursive type" +type b c // ERROR "invalid recursive type b\n.*b refers to c\n.*c refers to b|invalid recursive type|invalid recursive type" type c b // GCCGO_ERROR "invalid recursive type" type d e type e f -type f f // ERROR "invalid recursive type f\n\tLINE: f refers to\n\tLINE: f$|invalid recursive type" +type f f // ERROR "invalid recursive type: f refers to itself|invalid recursive type|invalid recursive type" -type g struct { // ERROR "invalid recursive type g\n\tLINE: g refers to\n\tLINE: g$|invalid recursive type" +type g struct { // ERROR "invalid recursive type: g refers to itself|invalid recursive type" h struct { g } } type w x -type x y // ERROR "invalid recursive type x\n\tLINE: x refers to\n\tLINE+1: y refers to\n\tLINE+2: z refers to\n\tLINE: x$|invalid recursive type" +type x y // ERROR "invalid recursive type x\n.*x refers to y\n.*y refers to z\n.*z refers to x|invalid recursive type" type y struct{ z } // GCCGO_ERROR "invalid recursive type" type z [10]x diff --git a/test/fixedbugs/issue48471.go b/test/fixedbugs/issue48471.go index 75875c40..a834c806 100644 --- a/test/fixedbugs/issue48471.go +++ b/test/fixedbugs/issue48471.go @@ -51,6 +51,6 @@ func g() { _ = i.(T6) // ERROR "impossible type assertion: i.\(T6\)\n\tT6 does not implement I \(missing method M\)\n\t\thave m\(int\) string\n\t\twant M\(int\)" var t *T4 - t = i // ERROR "cannot use i \(variable of type I\) as \*T4 value in assignment: need type assertion" + t = i // ERROR "cannot use i \(variable of interface type I\) as \*T4 value in assignment: need type assertion" _ = t } diff --git a/test/fixedbugs/issue50788.dir/b.go b/test/fixedbugs/issue50788.dir/b.go index e17afc7b..97ae2080 100644 --- a/test/fixedbugs/issue50788.dir/b.go +++ b/test/fixedbugs/issue50788.dir/b.go @@ -6,4 +6,4 @@ package b import "./a" -type T a.T[T] // ERROR "invalid recursive type T\n.*T refers to\n.*a\.T refers to\n.*T" +type T a.T[T] // ERROR "invalid recursive type T\n.*T refers to a\.T\n.*a\.T refers to T" diff --git a/test/fixedbugs/issue52193.go b/test/fixedbugs/issue52193.go index 32375d11..1c42210f 100644 --- a/test/fixedbugs/issue52193.go +++ b/test/fixedbugs/issue52193.go @@ -6,27 +6,6 @@ package p -import ( - "crypto/ecdh" - "crypto/rand" -) - -func F(peerShare []byte) ([]byte, error) { // ERROR "leaking param: peerShare" - p256 := ecdh.P256() // ERROR "inlining call to ecdh.P256" - - ourKey, err := p256.GenerateKey(rand.Reader) // ERROR "devirtualizing p256.GenerateKey" "inlining call to ecdh.*GenerateKey" - if err != nil { - return nil, err - } - - peerPublic, err := p256.NewPublicKey(peerShare) // ERROR "devirtualizing p256.NewPublicKey" "inlining call to ecdh.*NewPublicKey" - if err != nil { - return nil, err - } - - return ourKey.ECDH(peerPublic) -} - // Test that inlining doesn't break if devirtualization exposes a new // inlinable callee. diff --git a/test/fixedbugs/issue54159.go b/test/fixedbugs/issue54159.go index 8ef0e684..0f607b38 100644 --- a/test/fixedbugs/issue54159.go +++ b/test/fixedbugs/issue54159.go @@ -11,6 +11,7 @@ func run() { // ERROR "cannot inline run: recursive" g() // ERROR "inlining call to g" } f() // ERROR "inlining call to run.func1" "inlining call to g" + _ = f run() } diff --git a/test/fixedbugs/issue67329.go b/test/fixedbugs/issue67329.go new file mode 100644 index 00000000..5595c31c --- /dev/null +++ b/test/fixedbugs/issue67329.go @@ -0,0 +1,27 @@ +// errorcheck -0 -d=ssa/check_bce/debug=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x + +func Found(x []string) string { + switch len(x) { + default: + return x[0] + case 0, 1: + return "" + } +} + +func NotFound(x []string) string { + switch len(x) { + default: + return x[0] + case 0: + return "" + case 1: + return "" + } +} diff --git a/test/fixedbugs/issue68292.go b/test/fixedbugs/issue68292.go new file mode 100644 index 00000000..2a0d8267 --- /dev/null +++ b/test/fixedbugs/issue68292.go @@ -0,0 +1,12 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func f[S any, T any](T) {} +func g() { + f(0) // ERROR "in call to f, cannot infer S \(declared at issue68292.go:9:8\)" +} diff --git a/test/fixedbugs/issue68526.dir/a/a.go b/test/fixedbugs/issue68526.dir/a/a.go new file mode 100644 index 00000000..83462c7f --- /dev/null +++ b/test/fixedbugs/issue68526.dir/a/a.go @@ -0,0 +1,16 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.aliastypeparams + +package a + +type A[T any] = struct{ F T } + +type B = struct{ F int } + +func F() B { + type a[T any] = struct{ F T } + return a[int]{} +} diff --git a/test/fixedbugs/issue68526.dir/main.go b/test/fixedbugs/issue68526.dir/main.go new file mode 100644 index 00000000..966efd71 --- /dev/null +++ b/test/fixedbugs/issue68526.dir/main.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.aliastypeparams + +package main + +import ( + "fmt" + + "issue68526.dir/a" +) + +func main() { + unexported() + exported() +} + +func unexported() { + var want struct{ F int } + + if any(want) != any(a.B{}) || any(want) != any(a.F()) { + panic("zero value of alias and concrete type not identical") + } +} + +func exported() { + var ( + astr a.A[string] + aint a.A[int] + ) + + if any(astr) != any(struct{ F string }{}) || any(aint) != any(struct{ F int }{}) { + panic("zero value of alias and concrete type not identical") + } + + if any(astr) == any(aint) { + panic("zero value of struct{ F string } and struct{ F int } are not distinct") + } + + if got := fmt.Sprintf("%T", astr); got != "struct { F string }" { + panic(got) + } +} diff --git a/test/fixedbugs/issue68526.go b/test/fixedbugs/issue68526.go new file mode 100644 index 00000000..3067aa76 --- /dev/null +++ b/test/fixedbugs/issue68526.go @@ -0,0 +1,7 @@ +// runindir -goexperiment aliastypeparams -gomodversion "1.23" + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ignored diff --git a/test/fixedbugs/issue68734.go b/test/fixedbugs/issue68734.go new file mode 100644 index 00000000..e60dbc54 --- /dev/null +++ b/test/fixedbugs/issue68734.go @@ -0,0 +1,17 @@ +// compile + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The gofrontend had a bug handling panic of an untyped constant expression. + +package issue68734 + +func F1() { + panic(1 + 2) +} + +func F2() { + panic("a" + "b") +} diff --git a/test/fixedbugs/issue68809.go b/test/fixedbugs/issue68809.go new file mode 100644 index 00000000..67afda67 --- /dev/null +++ b/test/fixedbugs/issue68809.go @@ -0,0 +1,19 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func main() { + cnt := 0 + for i := 1; i <= 11; i++ { + if i-6 > 4 { + cnt++ + } + } + if cnt != 1 { + panic("bad") + } +} diff --git a/test/fixedbugs/issue68816.go b/test/fixedbugs/issue68816.go new file mode 100644 index 00000000..8622c9aa --- /dev/null +++ b/test/fixedbugs/issue68816.go @@ -0,0 +1,41 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func main() { + mustPanic(func() { + f1(1) + }) + f2(1, 0) // must not panic + mustPanic(func() { + f2(1, 2) + }) +} + +var v []func() + +//go:noinline +func f1(i int) { + v = make([]func(), -2|i) +} + +//go:noinline +func f2(i, j int) { + if j > 0 { + v = make([]func(), -2|i) + } +} + +func mustPanic(f func()) { + defer func() { + r := recover() + if r == nil { + panic("didn't panic") + } + }() + f() +} diff --git a/test/fixedbugs/issue69110.go b/test/fixedbugs/issue69110.go index 71a4bcac..ab51d0b5 100644 --- a/test/fixedbugs/issue69110.go +++ b/test/fixedbugs/issue69110.go @@ -1,5 +1,7 @@ // run +//go:build !goexperiment.swissmap + // Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/test/fixedbugs/issue69434.go b/test/fixedbugs/issue69434.go index 68204660..026d3246 100644 --- a/test/fixedbugs/issue69434.go +++ b/test/fixedbugs/issue69434.go @@ -7,167 +7,55 @@ package main import ( - "bufio" - "fmt" - "io" "iter" - "math/rand" - "os" - "strings" - "unicode" ) -// WordReader is the struct that implements io.Reader -type WordReader struct { - scanner *bufio.Scanner -} - -// NewWordReader creates a new WordReader from an io.Reader -func NewWordReader(r io.Reader) *WordReader { - scanner := bufio.NewScanner(r) - scanner.Split(bufio.ScanWords) - return &WordReader{ - scanner: scanner, - } -} - -// Read reads data from the input stream and returns a single lowercase word at a time -func (wr *WordReader) Read(p []byte) (n int, err error) { - if !wr.scanner.Scan() { - if err := wr.scanner.Err(); err != nil { - return 0, err - } - return 0, io.EOF - } - word := wr.scanner.Text() - cleanedWord := removeNonAlphabetic(word) - if len(cleanedWord) == 0 { - return wr.Read(p) - } - n = copy(p, []byte(cleanedWord)) - return n, nil -} - -// All returns an iterator allowing the caller to iterate over the WordReader using for/range. -func (wr *WordReader) All() iter.Seq[string] { - word := make([]byte, 1024) - return func(yield func(string) bool) { - var err error - var n int - for n, err = wr.Read(word); err == nil; n, err = wr.Read(word) { - if !yield(string(word[:n])) { +func All() iter.Seq[int] { + return func(yield func(int) bool) { + for i := 0; i < 10; i++ { + growStack(512) + if !yield(i) { return } } - if err != io.EOF { - fmt.Fprintf(os.Stderr, "error reading word: %v\n", err) - } } } -// removeNonAlphabetic removes non-alphabetic characters from a word using strings.Map -func removeNonAlphabetic(word string) string { - return strings.Map(func(r rune) rune { - if unicode.IsLetter(r) { - return unicode.ToLower(r) - } - return -1 - }, word) +type S struct { + round int } -// ProbabilisticSkipper determines if an item should be retained with probability 1/(1<>= 1 - pr.counter-- - if pr.counter == 0 { - pr.refreshCounter() - } - return remove -} - -// EstimateUniqueWordsIter estimates the number of unique words using a probabilistic counting method -func EstimateUniqueWordsIter(reader io.Reader, memorySize int) int { - wordReader := NewWordReader(reader) - words := make(map[string]struct{}, memorySize) - +func f() { rounds := 0 - roundRemover := NewProbabilisticSkipper(1) - wordSkipper := NewProbabilisticSkipper(rounds) - wordSkipper.check(rounds) + s := NewS(rounds) + s.check(rounds) - for word := range wordReader.All() { - wordSkipper.check(rounds) - if wordSkipper.ShouldSkip() { - delete(words, word) - } else { - words[word] = struct{}{} - - if len(words) >= memorySize { - rounds++ - - wordSkipper = NewProbabilisticSkipper(rounds) - for w := range words { - if roundRemover.ShouldSkip() { - delete(words, w) - } - } - } - } - wordSkipper.check(rounds) + for range All() { + s.check(rounds) + rounds++ + s = NewS(rounds) + s.check(rounds) } +} - if len(words) == 0 { - return 0 +func growStack(i int) { + if i == 0 { + return } - - invProbability := 1 << rounds - estimatedUniqueWords := len(words) * invProbability - return estimatedUniqueWords + growStack(i - 1) } func main() { - input := "Hello, world! This is a test. Hello, world, hello!" - expectedUniqueWords := 6 // "hello", "world", "this", "is", "a", "test" (but "hello" and "world" are repeated) - memorySize := 6 - - reader := strings.NewReader(input) - estimatedUniqueWords := EstimateUniqueWordsIter(reader, memorySize) - if estimatedUniqueWords != expectedUniqueWords { - // ... - } + f() } diff --git a/test/fixedbugs/issue69825.go b/test/fixedbugs/issue69825.go new file mode 100644 index 00000000..d3d3ec5e --- /dev/null +++ b/test/fixedbugs/issue69825.go @@ -0,0 +1,18 @@ +// compile -d=libfuzzer + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type T struct { + A +} + +type A struct { +} + +//go:noinline +func (a *A) Foo(s [2]string) { +} diff --git a/test/fixedbugs/issue70156.go b/test/fixedbugs/issue70156.go new file mode 100644 index 00000000..30a732a2 --- /dev/null +++ b/test/fixedbugs/issue70156.go @@ -0,0 +1,23 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "reflect" +) + +func main() { + pi := new(interface{}) + v := reflect.ValueOf(pi).Elem() + if v.Kind() != reflect.Interface { + panic(0) + } + if (v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface) && v.IsNil() { + return + } + panic(1) +} diff --git a/test/fixedbugs/issue70175.go b/test/fixedbugs/issue70175.go new file mode 100644 index 00000000..0ad6e39a --- /dev/null +++ b/test/fixedbugs/issue70175.go @@ -0,0 +1,17 @@ +// compile + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func f() { +_: + +_: +} + +func main() { + f() +} diff --git a/test/fixedbugs/issue70189.go b/test/fixedbugs/issue70189.go new file mode 100644 index 00000000..357ac537 --- /dev/null +++ b/test/fixedbugs/issue70189.go @@ -0,0 +1,38 @@ +// run -goexperiment noswissmap + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func nan() float64 { + var x, y float64 + return x / y +} + +func main() { + m := map[float64]int{} + + // Make a small map with nan keys + for i := 0; i < 8; i++ { + m[nan()] = i + } + + // Start iterating on it. + start := true + for _, v := range m { + if start { + // Add some more elements. + for i := 0; i < 10; i++ { + m[float64(i)] = i + } + // Now clear the map. + clear(m) + start = false + } else { + // We should never reach here. + panic(v) + } + } +} diff --git a/test/fixedbugs/issue70481.go b/test/fixedbugs/issue70481.go new file mode 100644 index 00000000..b73df851 --- /dev/null +++ b/test/fixedbugs/issue70481.go @@ -0,0 +1,20 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +const maxUint64 = (1 << 64) - 1 + +//go:noinline +func f(n uint64) uint64 { + return maxUint64 - maxUint64%n +} + +func main() { + for i := uint64(1); i < 20; i++ { + println(i, maxUint64-f(i)) + } +} diff --git a/test/fixedbugs/issue70481.out b/test/fixedbugs/issue70481.out new file mode 100644 index 00000000..bd41333d --- /dev/null +++ b/test/fixedbugs/issue70481.out @@ -0,0 +1,19 @@ +1 0 +2 1 +3 0 +4 3 +5 0 +6 3 +7 1 +8 7 +9 6 +10 5 +11 4 +12 3 +13 2 +14 1 +15 0 +16 15 +17 0 +18 15 +19 16 diff --git a/test/fixedbugs/notinheap.go b/test/fixedbugs/notinheap.go index 2188a38a..c94f2d6b 100644 --- a/test/fixedbugs/notinheap.go +++ b/test/fixedbugs/notinheap.go @@ -4,12 +4,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Test type-checking errors for go:notinheap. +// Test type-checking errors for not-in-heap types. + +//go:build cgo package p -//go:notinheap -type nih struct{} +import "runtime/cgo" + +type nih struct{ _ cgo.Incomplete } type embed4 map[nih]int // ERROR "incomplete \(or unallocatable\) map key not allowed" @@ -27,25 +30,8 @@ type okay4 interface { f(x nih) nih } -// Type conversions don't let you sneak past notinheap. - -type t1 struct{ x int } - -//go:notinheap -type t2 t1 - -//go:notinheap -type t3 byte - -//go:notinheap -type t4 rune - -var sink interface{} - -func i() { - sink = new(t1) // no error - sink = (*t2)(new(t1)) // ERROR "cannot convert(.|\n)*t2 is incomplete \(or unallocatable\)" - sink = (*t2)(new(struct{ x int })) // ERROR "cannot convert(.|\n)*t2 is incomplete \(or unallocatable\)" - sink = []t3("foo") // ERROR "cannot convert(.|\n)*t3 is incomplete \(or unallocatable\)" - sink = []t4("bar") // ERROR "cannot convert(.|\n)*t4 is incomplete \(or unallocatable\)" +func f() { + type embed7 map[nih]int // ERROR "incomplete \(or unallocatable\) map key not allowed" + type embed8 map[int]nih // ERROR "incomplete \(or unallocatable\) map value not allowed" + type emebd9 chan nih // ERROR "chan of incomplete \(or unallocatable\) type not allowed" } diff --git a/test/func3.go b/test/func3.go index 6be3bf01..861ab2cb 100644 --- a/test/func3.go +++ b/test/func3.go @@ -13,8 +13,8 @@ type t1 int type t2 int type t3 int -func f1(*t2, x t3) // ERROR "named" -func f2(t1, *t2, x t3) // ERROR "named" -func f3() (x int, *string) // ERROR "named" +func f1(*t2, x t3) // ERROR "missing parameter name" +func f2(t1, *t2, x t3) // ERROR "missing parameter name" +func f3() (x int, *string) // ERROR "missing parameter name" func f4() (t1 t1) // legal - scope of parameter named t1 starts in body of f4. diff --git a/test/fuse.go b/test/fuse.go index e9205dcc..9366b218 100644 --- a/test/fuse.go +++ b/test/fuse.go @@ -148,11 +148,11 @@ func fEqInterEqInter(a interface{}, f float64) bool { } func fEqInterNeqInter(a interface{}, f float64) bool { - return a == nil && f > Cf2 || a != nil && f < -Cf2 + return a == nil && f > Cf2 || a != nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqInterEqInter(a interface{}, f float64) bool { - return a != nil && f > Cf2 || a == nil && f < -Cf2 + return a != nil && f > Cf2 || a == nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqInterNeqInter(a interface{}, f float64) bool { @@ -164,11 +164,11 @@ func fEqSliceEqSlice(a []int, f float64) bool { } func fEqSliceNeqSlice(a []int, f float64) bool { - return a == nil && f > Cf2 || a != nil && f < -Cf2 + return a == nil && f > Cf2 || a != nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqSliceEqSlice(a []int, f float64) bool { - return a != nil && f > Cf2 || a == nil && f < -Cf2 + return a != nil && f > Cf2 || a == nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqSliceNeqSlice(a []int, f float64) bool { diff --git a/test/initloop.go b/test/initloop.go index b1a8470b..c4530f36 100644 --- a/test/initloop.go +++ b/test/initloop.go @@ -11,7 +11,7 @@ package main var ( x int = a - a int = b // ERROR "a refers to\n.*b refers to\n.*c refers to\n.*a|initialization loop" + a int = b // ERROR "a refers to b\n.*b refers to c\n.*c refers to a|initialization loop" b int = c c int = a ) diff --git a/test/inline.go b/test/inline.go index fd14f259..4714c795 100644 --- a/test/inline.go +++ b/test/inline.go @@ -73,6 +73,7 @@ func l(x, y int) (int, int, error) { // ERROR "can inline l" f := e f(nil) // ERROR "inlining call to l.func1" } + _ = e // prevent simple deadcode elimination after inlining return y, x, nil } @@ -109,6 +110,7 @@ func p() int { // ERROR "can inline p" func q(x int) int { // ERROR "can inline q" foo := func() int { return x * 2 } // ERROR "can inline q.func1" "func literal does not escape" + _ = foo // prevent simple deadcode elimination after inlining return foo() // ERROR "inlining call to q.func1" } @@ -121,6 +123,8 @@ func r(z int) int { return 2*y + x*z }(x) // ERROR "inlining call to r.func2.1" } + _, _ = foo, bar // prevent simple deadcode elimination after inlining + return foo(42) + bar(42) // ERROR "inlining call to r.func1" "inlining call to r.func2" "inlining call to r.r.func2.func3" } @@ -128,7 +132,8 @@ func s0(x int) int { // ERROR "can inline s0" foo := func() { // ERROR "can inline s0.func1" "func literal does not escape" x = x + 1 } - foo() // ERROR "inlining call to s0.func1" + foo() // ERROR "inlining call to s0.func1" + _ = foo // prevent simple deadcode elimination after inlining return x } @@ -137,6 +142,7 @@ func s1(x int) int { // ERROR "can inline s1" return x } x = x + 1 + _ = foo // prevent simple deadcode elimination after inlining return foo() // ERROR "inlining call to s1.func1" } diff --git a/test/inline_testingbloop.go b/test/inline_testingbloop.go new file mode 100644 index 00000000..cbdf9059 --- /dev/null +++ b/test/inline_testingbloop.go @@ -0,0 +1,37 @@ +// errorcheck -0 -m=2 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test no inlining of function calls in testing.B.Loop. +// See issue #61515. + +package foo + +import "testing" + +func caninline(x int) int { // ERROR "can inline caninline" + return x +} + +func cannotinline(b *testing.B) { // ERROR "b does not escape" "cannot inline cannotinline.*" + for i := 0; i < b.N; i++ { + caninline(1) // ERROR "inlining call to caninline" + } + for b.Loop() { // ERROR "skip inlining within testing.B.loop" "inlining call to testing\.\(\*B\)\.Loop" + caninline(1) + } + for i := 0; i < b.N; i++ { + caninline(1) // ERROR "inlining call to caninline" + } + for b.Loop() { // ERROR "skip inlining within testing.B.loop" "inlining call to testing\.\(\*B\)\.Loop" + caninline(1) + } + for i := 0; i < b.N; i++ { + caninline(1) // ERROR "inlining call to caninline" + } + for b.Loop() { // ERROR "skip inlining within testing.B.loop" "inlining call to testing\.\(\*B\)\.Loop" + caninline(1) + } +} diff --git a/test/internal/runtime/sys/README b/test/internal/runtime/sys/README new file mode 100644 index 00000000..919e0bb0 --- /dev/null +++ b/test/internal/runtime/sys/README @@ -0,0 +1,7 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +The internal/runtime/sys directory contains tests that specifically need to be +compiled as-if in the internal/runtime/sys package. For error-check tests, +these require the additional flags -+ and -p=internal/runtime/sys. diff --git a/test/runtime/inlinegcpc.go b/test/internal/runtime/sys/inlinegcpc.go similarity index 54% rename from test/runtime/inlinegcpc.go rename to test/internal/runtime/sys/inlinegcpc.go index c423993b..7cadc639 100644 --- a/test/runtime/inlinegcpc.go +++ b/test/internal/runtime/sys/inlinegcpc.go @@ -1,19 +1,19 @@ -// errorcheck -0 -+ -p=runtime -m +// errorcheck -0 -+ -p=internal/runtime/sys -m -// Copyright 2019 The Go Authors. All rights reserved. +// Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package runtime +package sys -// A function that calls runtime.getcallerpc or runtime.getcallersp() +// A function that calls sys.GetCallerPC or sys.GetCallerSP // cannot be inlined, no matter how small it is. -func getcallerpc() uintptr -func getcallersp() uintptr +func GetCallerPC() uintptr +func GetCallerSP() uintptr func pc() uintptr { - return getcallerpc() + 1 + return GetCallerPC() + 1 } func cpc() uintptr { // ERROR "can inline cpc" @@ -21,7 +21,7 @@ func cpc() uintptr { // ERROR "can inline cpc" } func sp() uintptr { - return getcallersp() + 3 + return GetCallerSP() + 3 } func csp() uintptr { // ERROR "can inline csp" diff --git a/test/intrinsic.dir/main.go b/test/intrinsic.dir/main.go index 951aa159..21e3eff8 100644 --- a/test/intrinsic.dir/main.go +++ b/test/intrinsic.dir/main.go @@ -6,7 +6,7 @@ package main import ( "fmt" - T "runtime/internal/sys" + T "internal/runtime/sys" ) var A = []uint64{0x0102030405060708, 0x1122334455667788} diff --git a/test/live.go b/test/live.go index 5658c8ba..c0b0fcd2 100644 --- a/test/live.go +++ b/test/live.go @@ -458,14 +458,14 @@ func f28(b bool) { func f29(b bool) { if b { - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hiter$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ (runtime.hiter|internal/runtime/maps.Iter)$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } @@ -659,15 +659,9 @@ func newT40() *T40 { return &ret } -func bad40() { - t := newT40() - _ = t - printnl() -} - func good40() { ret := T40{} // ERROR "stack object ret T40$" - ret.m = make(map[int]int) // ERROR "live at call to rand32: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hmap$" + ret.m = make(map[int]int) // ERROR "live at call to rand(32)?: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ (runtime.hmap|internal/runtime/maps.Map)$" t := &ret printnl() // ERROR "live at call to printnl: ret$" // Note: ret is live at the printnl because the compiler moves &ret @@ -675,6 +669,12 @@ func good40() { useT40(t) } +func bad40() { + t := newT40() + _ = t + printnl() +} + func ddd1(x, y *int) { // ERROR "live at entry to ddd1: x y$" ddd2(x, y) // ERROR "stack object .autotmp_[0-9]+ \[2\]\*int$" printnl() diff --git a/test/live2.go b/test/live2.go index 2beac4f8..313caa6f 100644 --- a/test/live2.go +++ b/test/live2.go @@ -27,14 +27,14 @@ func newT40() *T40 { } func bad40() { - t := newT40() // ERROR "stack object ret T40$" "stack object .autotmp_[0-9]+ runtime.hmap$" + t := newT40() // ERROR "stack object ret T40$" "stack object .autotmp_[0-9]+ (runtime.hmap|internal/runtime/maps.Map)$" printnl() // ERROR "live at call to printnl: ret$" useT40(t) } func good40() { ret := T40{} // ERROR "stack object ret T40$" - ret.m = make(map[int]int, 42) // ERROR "stack object .autotmp_[0-9]+ runtime.hmap$" + ret.m = make(map[int]int, 42) // ERROR "stack object .autotmp_[0-9]+ (runtime.hmap|internal/runtime/maps.Map)$" t := &ret printnl() // ERROR "live at call to printnl: ret$" useT40(t) diff --git a/test/live_regabi.go b/test/live_regabi.go index a335126b..35f874ec 100644 --- a/test/live_regabi.go +++ b/test/live_regabi.go @@ -278,6 +278,7 @@ func f17b(p *byte) { // ERROR "live at entry to f17b: p$" // key temporary if b { m2s[str()] = p // ERROR "live at call to mapassign_faststr: p$" "live at call to str: p$" + } m2s[str()] = p // ERROR "live at call to mapassign_faststr: p$" "live at call to str: p$" m2s[str()] = p // ERROR "live at call to mapassign_faststr: p$" "live at call to str: p$" @@ -455,14 +456,14 @@ func f28(b bool) { func f29(b bool) { if b { - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hiter$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ (runtime.hiter|internal/runtime/maps.Iter)$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } @@ -656,15 +657,9 @@ func newT40() *T40 { return &ret } -func bad40() { - t := newT40() - _ = t - printnl() -} - func good40() { ret := T40{} // ERROR "stack object ret T40$" - ret.m = make(map[int]int) // ERROR "live at call to rand32: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hmap$" + ret.m = make(map[int]int) // ERROR "live at call to rand(32)?: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ (runtime.hmap|internal/runtime/maps.Map)$" t := &ret printnl() // ERROR "live at call to printnl: ret$" // Note: ret is live at the printnl because the compiler moves &ret @@ -672,6 +667,12 @@ func good40() { useT40(t) } +func bad40() { + t := newT40() + _ = t + printnl() +} + func ddd1(x, y *int) { // ERROR "live at entry to ddd1: x y$" ddd2(x, y) // ERROR "stack object .autotmp_[0-9]+ \[2\]\*int$" printnl() diff --git a/test/loopbce.go b/test/loopbce.go index 04c186be..2d5c965a 100644 --- a/test/loopbce.go +++ b/test/loopbce.go @@ -27,7 +27,7 @@ func f0c(a []int) int { x := 0 for i := range a { // ERROR "Induction variable: limits \[0,\?\), increment 1$" b := a[:i+1] // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - x += b[0] + x += b[0] // ERROR "(\([0-9]+\) )?Proved IsInBounds$" } return x } @@ -168,7 +168,7 @@ func g2() int { func g3a() { a := "this string has length 25" for i := 0; i < len(a); i += 5 { // ERROR "Induction variable: limits \[0,20\], increment 5$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" useString(a[:i+3]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" useString(a[:i+5]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" useString(a[:i+6]) @@ -294,8 +294,10 @@ func k3neg2(a [100]int) [100]int { } func k4(a [100]int) [100]int { - min := (-1) << 63 - for i := min; i < min+50; i++ { // ERROR "Induction variable: limits \[-9223372036854775808,-9223372036854775758\), increment 1$" + // Note: can't use (-1)<<63 here, because i-min doesn't get rewritten to i+(-min), + // and it isn't worth adding that special case to prove. + min := (-1)<<63 + 1 + for i := min; i < min+50; i++ { // ERROR "Induction variable: limits \[-9223372036854775807,-9223372036854775757\), increment 1$" a[i-min] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" } return a @@ -314,7 +316,7 @@ func d1(a [100]int) [100]int { for i := 0; i < 100; i++ { // ERROR "Induction variable: limits \[0,100\), increment 1$" for j := 0; j < i; j++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" a[j] = 0 // ERROR "Proved IsInBounds$" - a[j+1] = 0 // FIXME: this boundcheck should be eliminated + a[j+1] = 0 // ERROR "Proved IsInBounds$" a[j+2] = 0 } } @@ -325,7 +327,7 @@ func d2(a [100]int) [100]int { for i := 0; i < 100; i++ { // ERROR "Induction variable: limits \[0,100\), increment 1$" for j := 0; i > j; j++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" a[j] = 0 // ERROR "Proved IsInBounds$" - a[j+1] = 0 // FIXME: this boundcheck should be eliminated + a[j+1] = 0 // ERROR "Proved IsInBounds$" a[j+2] = 0 } } @@ -419,12 +421,12 @@ func nobce2(a string) { for i := int64(0); i < int64(len(a))-31337; i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" } - for i := int64(0); i < int64(len(a))+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + for i := int64(0); i < int64(len(a))+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" "Disproved Less64" + useString(a[i:]) } j := int64(len(a)) - 123 - for i := int64(0); i < j+123+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + for i := int64(0); i < j+123+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" "Disproved Less64" + useString(a[i:]) } for i := int64(0); i < j+122+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" // len(a)-123+122+MinInt overflows when len(a) == 0, so a bound check is needed here diff --git a/test/newinline.go b/test/newinline.go index 69f1310a..a7288691 100644 --- a/test/newinline.go +++ b/test/newinline.go @@ -73,6 +73,7 @@ func l(x, y int) (int, int, error) { // ERROR "can inline l" f := e f(nil) // ERROR "inlining call to l.func1" } + _ = e // prevent simple deadcode elimination return y, x, nil } @@ -109,6 +110,7 @@ func p() int { // ERROR "can inline p" func q(x int) int { // ERROR "can inline q" foo := func() int { return x * 2 } // ERROR "can inline q.func1" "func literal does not escape" + _ = foo // prevent simple deadcode elimination return foo() // ERROR "inlining call to q.func1" } @@ -121,6 +123,8 @@ func r(z int) int { // ERROR "can inline r" return 2*y + x*z }(x) // ERROR "inlining call to r.func2.1" } + _ = foo // prevent simple deadcode elimination + _ = bar // prevent simple deadcode elimination return foo(42) + bar(42) // ERROR "inlining call to r.func1" "inlining call to r.func2" "inlining call to r.r.func2.func3" } @@ -128,7 +132,8 @@ func s0(x int) int { // ERROR "can inline s0" foo := func() { // ERROR "can inline s0.func1" "func literal does not escape" x = x + 1 } - foo() // ERROR "inlining call to s0.func1" + foo() // ERROR "inlining call to s0.func1" + _ = foo // prevent simple deadcode elimination return x } @@ -137,6 +142,7 @@ func s1(x int) int { // ERROR "can inline s1" return x } x = x + 1 + _ = foo // prevent simple deadcode elimination return foo() // ERROR "inlining call to s1.func1" } diff --git a/test/nosplit.go b/test/nosplit.go index e171d1da..4b4c93b1 100644 --- a/test/nosplit.go +++ b/test/nosplit.go @@ -1,6 +1,6 @@ // run -//go:build !nacl && !js && !aix && !wasip1 && !gcflags_noopt && gc +//go:build !nacl && !js && !aix && !openbsd && !wasip1 && !gcflags_noopt && gc // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/prove.go b/test/prove.go index 0d93db90..edfd8908 100644 --- a/test/prove.go +++ b/test/prove.go @@ -8,7 +8,10 @@ package main -import "math" +import ( + "math" + "math/bits" +) func f0(a []int) int { a[0] = 1 @@ -400,8 +403,8 @@ func f13f(a, b int64) int64 { if b != math.MaxInt64 { return 42 } - if a > b { - if a == 0 { // ERROR "Disproved Eq64$" + if a > b { // ERROR "Disproved Less64$" + if a == 0 { return 1 } } @@ -684,20 +687,6 @@ func constsuffix(s string) bool { return suffix(s, "abc") // ERROR "Proved IsSliceInBounds$" } -// oforuntil tests the pattern created by OFORUNTIL blocks. These are -// handled by addLocalInductiveFacts rather than findIndVar. -func oforuntil(b []int) { - i := 0 - if len(b) > i { - top: - println(b[i]) // ERROR "Induction variable: limits \[0,\?\), increment 1$" "Proved IsInBounds$" - i++ - if i < len(b) { - goto top - } - } -} - func atexit(foobar []func()) { for i := len(foobar) - 1; i >= 0; i-- { // ERROR "Induction variable: limits \[0,\?\], increment 1" f := foobar[i] @@ -877,11 +866,11 @@ func unrollDecMin(a []int, b int) int { return 42 } var i, x int - for i = len(a); i >= b; i -= 2 { + for i = len(a); i >= b; i -= 2 { // ERROR "Proved Leq64" x += a[i-1] x += a[i-2] } - if i == 1 { // ERROR "Disproved Eq64$" + if i == 1 { x += a[i-1] } return x @@ -893,11 +882,11 @@ func unrollIncMin(a []int, b int) int { return 42 } var i, x int - for i = len(a); i >= b; i += 2 { + for i = len(a); i >= b; i += 2 { // ERROR "Proved Leq64" x += a[i-1] x += a[i-2] } - if i == 1 { // ERROR "Disproved Eq64$" + if i == 1 { x += a[i-1] } return x @@ -1107,7 +1096,7 @@ func modu2(x, y uint) int { func issue57077(s []int) (left, right []int) { middle := len(s) / 2 - left = s[:middle] // ERROR "Proved IsSliceInBounds$" + left = s[:middle] // ERROR "Proved IsSliceInBounds$" right = s[middle:] // ERROR "Proved IsSliceInBounds$" return } @@ -1124,6 +1113,605 @@ func issue45928(x int) { useInt(combinedFrac) } +func constantBounds1(i, j uint) int { + var a [10]int + if j < 11 && i < j { + return a[i] // ERROR "Proved IsInBounds$" + } + return 0 +} + +func constantBounds2(i, j uint) int { + var a [10]int + if i < j && j < 11 { + return a[i] // ERROR "Proved IsInBounds" + } + return 0 +} + +func constantBounds3(i, j, k, l uint) int { + var a [8]int + if i < j && j < k && k < l && l < 11 { + return a[i] // ERROR "Proved IsInBounds" + } + return 0 +} + +func equalityPropagation(a [1]int, i, j uint) int { + if i == j && i == 5 { + return a[j-5] // ERROR "Proved IsInBounds" + } + return 0 +} +func inequalityPropagation(a [1]int, i, j uint) int { + if i != j && j >= 5 && j <= 6 && i == 5 { + return a[j-6] // ERROR "Proved IsInBounds" + } + return 0 +} + +func issue66826a(a [21]byte) { + for i := 0; i <= 10; i++ { // ERROR "Induction variable: limits \[0,10\], increment 1$" + _ = a[2*i] // ERROR "Proved IsInBounds" + } +} +func issue66826b(a [31]byte, i int) { + if i < 0 || i > 10 { + return + } + _ = a[3*i] // ERROR "Proved IsInBounds" +} + +func f20(a, b bool) int { + if a == b { + if a { + if b { // ERROR "Proved Arg" + return 1 + } + } + } + return 0 +} + +func f21(a, b *int) int { + if a == b { + if a != nil { + if b != nil { // ERROR "Proved IsNonNil" + return 1 + } + } + } + return 0 +} + +func f22(b bool, x, y int) int { + b2 := x < y + if b == b2 { + if b { + if x >= y { // ERROR "Disproved Leq64$" + return 1 + } + } + } + return 0 +} + +func ctz64(x uint64, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint64 + sz := bits.Len64(max) + + log2half := uint64(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros64(x) // ERROR "Proved Ctz64 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} +func ctz32(x uint32, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint32 + sz := bits.Len32(max) + + log2half := uint32(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros32(x) // ERROR "Proved Ctz32 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} +func ctz16(x uint16, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint16 + sz := bits.Len16(max) + + log2half := uint16(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros16(x) // ERROR "Proved Ctz16 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} +func ctz8(x uint8, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint8 + sz := bits.Len8(max) + + log2half := uint8(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros8(x) // ERROR "Proved Ctz8 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} + +func bitLen64(x uint64, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint64 + sz := bits.Len64(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len64(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} +func bitLen32(x uint32, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint32 + sz := bits.Len32(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len32(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} +func bitLen16(x uint16, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint16 + sz := bits.Len16(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len16(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} +func bitLen8(x uint8, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint8 + sz := bits.Len8(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len8(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} + +func xor64(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a &= 0xff + b &= 0xfff + + z := a ^ b + + if ensureBothBranchesCouldHappen { + if z > 0xfff { // ERROR "Disproved Less64U$" + return 42 + } + } else { + if z <= 0xfff { // ERROR "Proved Leq64U$" + return 1337 + } + } + return int(z) +} + +func or64(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a &= 0xff + b &= 0xfff + + z := a | b + + if ensureBothBranchesCouldHappen { + if z > 0xfff { // ERROR "Disproved Less64U$" + return 42 + } + } else { + if z <= 0xfff { // ERROR "Proved Leq64U$" + return 1337 + } + } + return int(z) +} + +func mod64uWithSmallerDividendMax(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a &= 0xff + b &= 0xfff + + z := bits.Len64(a % b) // see go.dev/issue/68857 for bits.Len64 + + if ensureBothBranchesCouldHappen { + if z > bits.Len64(0xff) { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= bits.Len64(0xff) { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64uWithSmallerDivisorMax(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a &= 0xfff + b &= 0x10 // we need bits.Len64(b.umax) != bits.Len64(b.umax-1) + + z := bits.Len64(a % b) // see go.dev/issue/68857 for bits.Len64 + + if ensureBothBranchesCouldHappen { + if z > bits.Len64(0x10-1) { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= bits.Len64(0x10-1) { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64uWithIdenticalMax(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a &= 0x10 + b &= 0x10 // we need bits.Len64(b.umax) != bits.Len64(b.umax-1) + + z := bits.Len64(a % b) // see go.dev/issue/68857 for bits.Len64 + + if ensureBothBranchesCouldHappen { + if z > bits.Len64(0x10-1) { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= bits.Len64(0x10-1) { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64sPositiveWithSmallerDividendMax(a, b int64, ensureBothBranchesCouldHappen bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a &= 0xff + b &= 0xfff + + z := a % b // ERROR "Proved Mod64 does not need fix-up$" + + if ensureBothBranchesCouldHappen { + if z > 0xff { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= 0xff { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64sPositiveWithSmallerDivisorMax(a, b int64, ensureBothBranchesCouldHappen bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a &= 0xfff + b &= 0xff + + z := a % b // ERROR "Proved Mod64 does not need fix-up$" + + if ensureBothBranchesCouldHappen { + if z > 0xff-1 { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= 0xff-1 { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64sPositiveWithIdenticalMax(a, b int64, ensureBothBranchesCouldHappen bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a &= 0xfff + b &= 0xfff + + z := a % b // ERROR "Proved Mod64 does not need fix-up$" + + if ensureBothBranchesCouldHappen { + if z > 0xfff-1 { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= 0xfff-1 { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} + +func div64u(a, b uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + a &= 0xffff + a |= 0xfff + b &= 0xff + b |= 0xf + + z := a / b // ERROR "Proved Neq64$" + + if ensureAllBranchesCouldHappen() && z > 0xffff/0xf { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= 0xffff/0xf { // ERROR "Proved Leq64U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < 0xfff/0xff { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= 0xfff/0xff { // ERROR "Proved Leq64U$" + return 42 + } + return z +} +func div64s(a, b int64, ensureAllBranchesCouldHappen func() bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a &= 0xffff + a |= 0xfff + b &= 0xff + b |= 0xf + + z := a / b // ERROR "(Proved Div64 does not need fix-up|Proved Neq64)$" + + if ensureAllBranchesCouldHappen() && z > 0xffff/0xf { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= 0xffff/0xf { // ERROR "Proved Leq64$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < 0xfff/0xff { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= 0xfff/0xff { // ERROR "Proved Leq64$" + return 42 + } + return z +} + +func trunc64to16(a uint64, ensureAllBranchesCouldHappen func() bool) uint16 { + a &= 0xfff + a |= 0xff + + z := uint16(a) + if ensureAllBranchesCouldHappen() && z > 0xfff { // ERROR "Disproved Less16U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= 0xfff { // ERROR "Proved Leq16U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < 0xff { // ERROR "Disproved Less16U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= 0xff { // ERROR "Proved Leq16U$" + return 1337 + } + return z +} + +func com64(a uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + a &= 0xffff + a |= 0xff + + z := ^a + + if ensureAllBranchesCouldHappen() && z > ^uint64(0xff) { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= ^uint64(0xff) { // ERROR "Proved Leq64U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < ^uint64(0xffff) { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= ^uint64(0xffff) { // ERROR "Proved Leq64U$" + return 1337 + } + return z +} + +func neg64(a uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + var lo, hi uint64 = 0xff, 0xfff + a &= hi + a |= lo + + z := -a + + if ensureAllBranchesCouldHappen() && z > -lo { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= -lo { // ERROR "Proved Leq64U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < -hi { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= -hi { // ERROR "Proved Leq64U$" + return 1337 + } + return z +} +func neg64mightOverflowDuringNeg(a uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + var lo, hi uint64 = 0, 0xfff + a &= hi + a |= lo + + z := -a + + if ensureAllBranchesCouldHappen() && z > -lo { + return 42 + } + if ensureAllBranchesCouldHappen() && z <= -lo { + return 1337 + } + if ensureAllBranchesCouldHappen() && z < -hi { + return 42 + } + if ensureAllBranchesCouldHappen() && z >= -hi { + return 1337 + } + return z +} + +func phiMin(a, b []byte) { + _ = a[:min(len(a), len(b))] // ERROR "Proved IsSliceInBounds" + _ = b[:min(len(a), len(b))] // ERROR "Proved IsSliceInBounds" + _ = a[:max(len(a), len(b))] + _ = b[:max(len(a), len(b))] + x := len(a) + if x > len(b) { + x = len(b) + useInt(0) + } + _ = a[:x] // ERROR "Proved IsSliceInBounds" + y := len(a) + if y > len(b) { + y = len(b) + useInt(0) + } else { + useInt(1) + } + _ = b[:y] // ERROR "Proved IsSliceInBounds" +} + +func issue16833(a, b []byte) { + n := copy(a, b) + _ = a[n:] // ERROR "Proved IsSliceInBounds" + _ = b[n:] // ERROR "Proved IsSliceInBounds" + _ = a[:n] // ERROR "Proved IsSliceInBounds" + _ = b[:n] // ERROR "Proved IsSliceInBounds" +} + +func clampedIdx1(x []int, i int) int { + if len(x) == 0 { + return 0 + } + return x[min(max(0, i), len(x)-1)] // ERROR "Proved IsInBounds" +} +func clampedIdx2(x []int, i int) int { + if len(x) == 0 { + return 0 + } + return x[max(min(i, len(x)-1), 0)] // TODO: can't get rid of this bounds check yet +} + //go:noinline func useInt(a int) { } diff --git a/test/prove_constant_folding.go b/test/prove_constant_folding.go index ed63e685..bca96847 100644 --- a/test/prove_constant_folding.go +++ b/test/prove_constant_folding.go @@ -14,7 +14,7 @@ func f0i(x int) int { } if (x + 20) == 20 { - return x + 5 // ERROR "Proved.+is constant 0$" + return x + 5 // ERROR "Proved.+is constant 0$" "Proved.+is constant 5$" } return x / 2 @@ -26,7 +26,7 @@ func f0u(x uint) uint { } if (x + 20) == 20 { - return x + 5 // ERROR "Proved.+is constant 0$" + return x + 5 // ERROR "Proved.+is constant 0$" "Proved.+is constant 5$" } return x / 2 diff --git a/test/runtime/README b/test/runtime/README deleted file mode 100644 index 249031af..00000000 --- a/test/runtime/README +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -The runtime directory contains tests that specifically need -to be compiled as-if in the runtime package. For error-check -tests, these require the additional flags -+ and -p=runtime. diff --git a/test/switch2.go b/test/switch2.go index 66e89fda..113d81be 100644 --- a/test/switch2.go +++ b/test/switch2.go @@ -25,12 +25,12 @@ func f() { switch { case 0: f(); case 0: - case 0: f() case 0: // ERROR "unexpected case at end of statement" + case 0: f() case 0: // ERROR "unexpected keyword case at end of statement" } switch { case 0: f(); default: - case 0: f() default: // ERROR "unexpected default at end of statement" + case 0: f() default: // ERROR "unexpected keyword default at end of statement" } switch { diff --git a/test/syntax/semi7.go b/test/syntax/semi7.go index a1948b0f..acd2f835 100644 --- a/test/syntax/semi7.go +++ b/test/syntax/semi7.go @@ -8,7 +8,7 @@ package main func main() { if x { } // GCCGO_ERROR "undefined" - else { } // ERROR "unexpected semicolon or newline before .?else.?|unexpected else" + else { } // ERROR "unexpected semicolon or newline before .?else.?|unexpected keyword else" } diff --git a/test/tailcall.go b/test/tailcall.go new file mode 100644 index 00000000..6b14a2f1 --- /dev/null +++ b/test/tailcall.go @@ -0,0 +1,29 @@ +// errorcheck -0 -d=tailcall=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +// Test that when generating wrappers for methods, we generate a tail call to the pointer version of +// the method, if that method is not inlineable. We use go:noinline here to force the non-inlineability +// condition. + +//go:noinline +func (f *Foo) Get2Vals() [2]int { return [2]int{f.Val, f.Val + 1} } +func (f *Foo) Get3Vals() [3]int { return [3]int{f.Val, f.Val + 1, f.Val + 2} } + +type Foo struct{ Val int } + +type Bar struct { // ERROR "tail call emitted for the method \(\*Foo\).Get2Vals wrapper" + int64 + *Foo // needs a method wrapper + string +} + +var i any + +func init() { + i = Bar{1, nil, "first"} +} diff --git a/test/wasmexport.go b/test/wasmexport.go new file mode 100644 index 00000000..3b92ae93 --- /dev/null +++ b/test/wasmexport.go @@ -0,0 +1,19 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Verify that misplaced directives are diagnosed. + +//go:build wasm + +package p + +//go:wasmexport F +func F() {} // OK + +type S int32 + +//go:wasmexport M +func (S) M() {} // ERROR "cannot use //go:wasmexport on a method" diff --git a/test/wasmexport2.go b/test/wasmexport2.go new file mode 100644 index 00000000..cfbfab99 --- /dev/null +++ b/test/wasmexport2.go @@ -0,0 +1,92 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Verify that wasmexport supports allowed types and rejects +// unallowed types. + +//go:build wasm + +package p + +import ( + "structs" + "unsafe" +) + +//go:wasmexport good1 +func good1(int32, uint32, int64, uint64, float32, float64, unsafe.Pointer) {} // allowed types + +type MyInt32 int32 + +//go:wasmexport good2 +func good2(MyInt32) {} // named type is ok + +//go:wasmexport good3 +func good3() int32 { return 0 } // one result is ok + +//go:wasmexport good4 +func good4() unsafe.Pointer { return nil } // one result is ok + +//go:wasmexport good5 +func good5(string, uintptr) bool { return false } // bool, string, and uintptr are allowed + +//go:wasmexport bad1 +func bad1(any) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad2 +func bad2(func()) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad3 +func bad3(uint8) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad4 +func bad4(int) {} // ERROR "go:wasmexport: unsupported parameter type" + +// Struct and array types are also not allowed. + +type S struct { x, y int32 } + +type H struct { _ structs.HostLayout; x, y int32 } + +type A = structs.HostLayout + +type AH struct { _ A; x, y int32 } + +//go:wasmexport bad5 +func bad5(S) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad6 +func bad6(H) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad7 +func bad7([4]int32) {} // ERROR "go:wasmexport: unsupported parameter type" + +// Pointer types are not allowed, with resitrictions on +// the element type. + +//go:wasmexport good6 +func good6(*int32, *uint8, *bool) {} + +//go:wasmexport bad8 +func bad8(*S) {} // ERROR "go:wasmexport: unsupported parameter type" // without HostLayout, not allowed + +//go:wasmexport bad9 +func bad9() *S { return nil } // ERROR "go:wasmexport: unsupported result type" + +//go:wasmexport good7 +func good7(*H, *AH) {} // pointer to struct with HostLayout is allowed + +//go:wasmexport good8 +func good8(*struct{}) {} // pointer to empty struct is allowed + +//go:wasmexport good9 +func good9(*[4]int32, *[2]H) {} // pointer to array is allowed, if the element type is okay + +//go:wasmexport toomanyresults +func toomanyresults() (int32, int32) { return 0, 0 } // ERROR "go:wasmexport: too many return values" + +//go:wasmexport bad10 +func bad10() string { return "" } // ERROR "go:wasmexport: unsupported result type" // string cannot be a result diff --git a/test/wasmmemsize.dir/asm_wasm.s b/test/wasmmemsize.dir/asm_wasm.s new file mode 100644 index 00000000..daccfcc7 --- /dev/null +++ b/test/wasmmemsize.dir/asm_wasm.s @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT ·currentMemory(SB), NOSPLIT, $0 + Get SP + CurrentMemory + I32Store ret+0(FP) + RET diff --git a/test/wasmmemsize.dir/main.go b/test/wasmmemsize.dir/main.go new file mode 100644 index 00000000..d3beab24 --- /dev/null +++ b/test/wasmmemsize.dir/main.go @@ -0,0 +1,30 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "io" +) + +// Expect 8 MB of memory usage for a small wasm program. +// This reflects the current allocator. We test an exact +// value here, but if the allocator changes, we can update +// or relax this. +const want = 8 << 20 + +var w = io.Discard + +func main() { + fmt.Fprintln(w, "hello world") + + const pageSize = 64 * 1024 + sz := uintptr(currentMemory()) * pageSize + if sz != want { + fmt.Printf("FAIL: unexpected memory size %d, want %d\n", sz, want) + } +} + +func currentMemory() int32 // implemented in assembly diff --git a/test/wasmmemsize.go b/test/wasmmemsize.go new file mode 100644 index 00000000..44981690 --- /dev/null +++ b/test/wasmmemsize.go @@ -0,0 +1,11 @@ +// runindir + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test checks the memory size of a small wasm program. + +//go:build wasm + +package ignored diff --git a/test/weak.go b/test/weak.go new file mode 100644 index 00000000..ca3ec797 --- /dev/null +++ b/test/weak.go @@ -0,0 +1,24 @@ +// errorcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test weak pointers. + +package p + +import ( + "runtime" + "weak" +) + +// Adapted from example in https://github.com/golang/go/issues/67552#issuecomment-2639661220 +func conversion() { + p := "hello" + a := weak.Make(&p) + b := (weak.Pointer[*byte])(a) // ERROR "cannot convert a \(variable of struct type weak\.Pointer\[string\]\) to type weak.Pointer\[\*byte\]" + c := b.Value() + println(**c) + runtime.KeepAlive(p) +}